Skip to content

Commit

Permalink
Backups: Add config option to limit the number of backups to keep #4243
Browse files Browse the repository at this point in the history 
PHOTOPRISM_BACKUP_RETAIN lets to specify the number of index database
dumps to keep (backup filenames are in the format "YYYY-MM-DD.sql").

Signed-off-by: Michael Mayer <michael@photoprism.app>
  • Loading branch information
@lastzero
lastzero committed May 12, 2024
1 parent 0e7c91f commit 1fe0bab
Show file tree
Hide file tree
Showing 10 changed files with 221 additions and 146 deletions.
23 changes: 15 additions & 8 deletions internal/commands/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/dustin/go-humanize/english"
"github.com/urfave/cli"

"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/pkg/clean"
"github.com/photoprism/photoprism/pkg/fs"
Expand All @@ -23,7 +24,7 @@ const backupDescription = "A user-defined filename or - for stdout can be passed
var BackupCommand = cli.Command{
Name: "backup",
Description: backupDescription,
Usage: "Creates an index backup and optionally album YAML files organized by type",
Usage: "Creates an index database dump and/or album YAML file backups",
ArgsUsage: "[filename]",
Flags: backupFlags,
Action: backupAction,
Expand All @@ -32,23 +33,28 @@ var BackupCommand = cli.Command{
var backupFlags = []cli.Flag{
cli.BoolFlag{
Name: "force, f",
Usage: "replace existing files",
Usage: "replace existing index backup files",
},
cli.BoolFlag{
Name: "albums, a",
Usage: "create album YAML files organized by type",
Usage: "create album YAML file backups in the configured backup path",
},
cli.StringFlag{
Name: "albums-path",
Usage: "custom album files `PATH`",
Usage: "custom `PATH` for creating album backups",
},
cli.BoolFlag{
Name: "index, i",
Usage: "create index backup",
Usage: "create index backup in the configured backup path (stdout if - is passed as first argument)",
},
cli.StringFlag{
Name: "index-path",
Usage: "custom index backup `PATH`",
Usage: "custom `PATH` for creating index backups",
},
cli.IntFlag{
Name: "retain, r",
Usage: "`NUMBER` of index backups to keep (-1 to keep all)",
Value: config.DefaultBackupRetain,
},
}

Expand All @@ -61,6 +67,7 @@ func backupAction(ctx *cli.Context) error {
albumsPath := ctx.String("albums-path")
backupAlbums := ctx.Bool("albums") || albumsPath != ""
force := ctx.Bool("force")
retain := ctx.Int("retain")

if !backupIndex && !backupAlbums {
return cli.ShowSubcommandHelp(ctx)
Expand Down Expand Up @@ -95,8 +102,8 @@ func backupAction(ctx *cli.Context) error {
fileName = filepath.Join(backupPath, backupFile)
}

if err = photoprism.BackupIndex(backupPath, fileName, fileName == "-", force); err != nil {
return fmt.Errorf("failed to create %s: %w", clean.Log(fileName), err)
if err = photoprism.BackupIndex(backupPath, fileName, fileName == "-", force, retain); err != nil {
return fmt.Errorf("failed to create index backup: %w", err)
}
}

Expand Down
129 changes: 12 additions & 117 deletions internal/commands/restore.go
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,12 @@
package commands

import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"time"

"github.com/dustin/go-humanize/english"
"github.com/urfave/cli"

"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/get"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/pkg/clean"
Expand All @@ -30,7 +21,7 @@ const restoreDescription = "A user-defined filename or - for stdin can be passed
var RestoreCommand = cli.Command{
Name: "restore",
Description: restoreDescription,
Usage: "Restores the index from a backup and optionally albums from YAML files",
Usage: "Restores the index from a database dump and/or album YAML file backups",
ArgsUsage: "[filename]",
Flags: restoreFlags,
Action: restoreAction,
Expand All @@ -39,23 +30,23 @@ var RestoreCommand = cli.Command{
var restoreFlags = []cli.Flag{
cli.BoolFlag{
Name: "force, f",
Usage: "replace existing index",
Usage: "replace existing index schema and data",
},
cli.BoolFlag{
Name: "albums, a",
Usage: "restore albums from YAML files",
Usage: "restore album YAML file backups from the configured backup path",
},
cli.StringFlag{
Name: "albums-path",
Usage: "custom album files `PATH`",
Usage: "custom `PATH` for restoring album backups",
},
cli.BoolFlag{
Name: "index, i",
Usage: "restore index from backup",
Usage: "restore index from the latest backup in the configured backup path (or the file passed as first argument)",
},
cli.StringFlag{
Name: "index-path",
Usage: "custom index backup `PATH`",
Usage: "custom `PATH` for restoring index backups",
},
}

Expand All @@ -65,7 +56,7 @@ func restoreAction(ctx *cli.Context) error {
indexFileName := ctx.Args().First()
indexPath := ctx.String("index-path")
restoreIndex := ctx.Bool("index") || indexFileName != "" || indexPath != ""

force := ctx.Bool("force")
albumsPath := ctx.String("albums-path")
restoreAlbums := ctx.Bool("albums") || albumsPath != ""

Expand All @@ -87,107 +78,11 @@ func restoreAction(ctx *cli.Context) error {
conf.RegisterDb()
defer conf.Shutdown()

if restoreIndex {
// If empty, use default backup file name.
if indexFileName == "" {
if indexPath == "" {
indexPath = filepath.Join(conf.BackupPath(), conf.DatabaseDriver())
}

matches, err := filepath.Glob(filepath.Join(regexp.QuoteMeta(indexPath), "*.sql"))

if err != nil {
return err
}

if len(matches) == 0 {
log.Errorf("no backup files found in %s", indexPath)
return nil
}

indexFileName = matches[len(matches)-1]
}

counts := struct{ Photos int }{}

conf.Db().Unscoped().Table("photos").
Select("COUNT(*) AS photos").
Take(&counts)

if counts.Photos == 0 {
// Do nothing;
} else if !ctx.Bool("force") {
return fmt.Errorf("found exisisting index with %d pictures, use --force to replace it", counts.Photos)
} else {
log.Warnf("replacing existing index with %d pictures", counts.Photos)
}

tables := entity.Entities

var cmd *exec.Cmd

switch conf.DatabaseDriver() {
case config.MySQL, config.MariaDB:
cmd = exec.Command(
conf.MariadbBin(),
"--protocol", "tcp",
"-h", conf.DatabaseHost(),
"-P", conf.DatabasePortString(),
"-u", conf.DatabaseUser(),
"-p"+conf.DatabasePassword(),
"-f",
conf.DatabaseName(),
)
case config.SQLite3:
log.Infoln("dropping existing tables")
tables.Drop(conf.Db())
cmd = exec.Command(
conf.SqliteBin(),
conf.DatabaseFile(),
)
default:
return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver())
}

// Read from stdin or file.
var f *os.File
if indexFileName == "-" {
log.Infof("restoring index from stdin")
f = os.Stdin
} else if f, err = os.OpenFile(indexFileName, os.O_RDONLY, 0); err != nil {
return fmt.Errorf("failed to open %s: %s", clean.Log(indexFileName), err)
} else {
log.Infof("restoring index from %s", clean.Log(indexFileName))
defer f.Close()
}

var stderr bytes.Buffer
var stdin io.WriteCloser
cmd.Stderr = &stderr
cmd.Stdout = os.Stdout
stdin, err = cmd.StdinPipe()

if err != nil {
log.Fatal(err)
}

go func() {
defer stdin.Close()
if _, err = io.Copy(stdin, f); err != nil {
log.Errorf(err.Error())
}
}()

// Log exact command for debugging in trace mode.
log.Trace(cmd.String())

// Run backup command.
if err := cmd.Run(); err != nil {
if stderr.String() != "" {
log.Debugln(stderr.String())
log.Warnf("index could not be restored completely")
}
}
// Restore index from specified file?
if !restoreIndex {
// Do nothing.
} else if err = photoprism.RestoreIndex(indexPath, indexFileName, indexFileName == "-", force); err != nil {
return err
}

log.Infoln("migrating index database schema")
Expand Down
1 change: 1 addition & 0 deletions internal/commands/show_config_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ func showConfigOptionsAction(ctx *cli.Context) error {
{Start: "PHOTOPRISM_ADMIN_PASSWORD", Title: "Authentication"},
{Start: "PHOTOPRISM_LOG_LEVEL", Title: "Logging"},
{Start: "PHOTOPRISM_CONFIG_PATH", Title: "Storage"},
{Start: "PHOTOPRISM_BACKUP_PATH", Title: "Backups"},
{Start: "PHOTOPRISM_INDEX_WORKERS, PHOTOPRISM_WORKERS", Title: "Index Workers"},
{Start: "PHOTOPRISM_READONLY", Title: "Feature Flags"},
{Start: "PHOTOPRISM_DEFAULT_LOCALE", Title: "Customization"},
Expand Down
2 changes: 1 addition & 1 deletion internal/config/config_backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func (c *Config) BackupPath() string {
return filepath.Join(c.StoragePath(), "backup")
}

// BackupIndex checks if index SQL database dumps should be created based on the configured schedule.
// BackupIndex checks if SQL database dumps should be created based on the configured schedule.
func (c *Config) BackupIndex() bool {
return c.options.BackupIndex
}
Expand Down
2 changes: 1 addition & 1 deletion internal/config/config_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (c *Config) DatabaseDsn() string {

// DatabaseFile returns the filename part of a sqlite database DSN.
func (c *Config) DatabaseFile() string {
fileName, _, _ := strings.Cut(c.DatabaseDsn(), "?")
fileName, _, _ := strings.Cut(strings.TrimPrefix(c.DatabaseDsn(), "file:"), "?")
return fileName
}

Expand Down
6 changes: 3 additions & 3 deletions internal/config/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,12 +183,12 @@ var Flags = CliFlags{
}}, {
Flag: cli.StringFlag{
Name: "backup-path, ba",
Usage: "custom backup `PATH` for index backup files *optional*",
Usage: "custom default `PATH` for creating and restoring index backups *optional*",
EnvVar: EnvVar("BACKUP_PATH"),
}}, {
Flag: cli.BoolFlag{
Name: "backup-index",
Usage: "create index SQL database dumps based on the configured schedule",
Usage: "create index backups based on the configured schedule",
EnvVar: EnvVar("BACKUP_INDEX"),
}}, {
Flag: cli.BoolFlag{
Expand All @@ -198,7 +198,7 @@ var Flags = CliFlags{
}}, {
Flag: cli.IntFlag{
Name: "backup-retain",
Usage: "maximum `NUMBER` of SQL database dumps to keep (-1 to keep all)",
Usage: "`NUMBER` of index backups to keep (-1 to keep all)",
Value: DefaultBackupRetain,
EnvVar: EnvVar("BACKUP_RETAIN"),
}}, {
Expand Down

0 comments on commit 1fe0bab

Please sign in to comment.