diff --git a/README.md b/README.md index 0a0dc1e..7a613af 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ options of the PostgreSQL instance. * Purge based on age and number of dumps to keep * Dump from a hot standby by pausing replication replay * Encrypt and decrypt dumps and other files -* Upload dumps to S3, GCS, Azure or a remote host with SFTP +* Upload and download dumps to S3, GCS, Azure or a remote host with SFTP ## Install @@ -213,6 +213,20 @@ on the remote location as the local directory. When files are encrypted and their unencrypted source is kept, only encrypted files are uploaded. +### Downloading from remote locations + +Previously uploaded files can be downloaded using the `--download` option with +a value different than `none`, similarly to `--upload`. The options to setup +the remote access are the same as `--upload`. + +When downloading files, dumps are not performed. Arguments on the commandline +(database names when dumping) are used as shell globs to choose which files to +the backup directory. + +If `--download` is used at the same time as `--decrypt`, files are downloaded +first, then files matching globs are decrypted. + + ## Restoring files The following files are created: diff --git a/config.go b/config.go index 63d623f..08ad0b1 100644 --- a/config.go +++ b/config.go @@ -83,6 +83,7 @@ type options struct { DumpOnly bool Upload string // values are none, s3, sftp, gcs + Download string // values are none, s3, sftp, gcs PurgeRemote bool S3Region string S3Bucket string @@ -132,6 +133,7 @@ func defaultOptions() options { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", } } @@ -284,6 +286,7 @@ func parseCli(args []string) (options, []string, error) { pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with 'AGE-SECRET-KEY-1'\n") pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload") + pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files") purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory") pflag.StringVar(&opts.S3Region, "s3-region", "", "S3 region") @@ -438,35 +441,41 @@ func parseCli(args []string) (options, []string, error) { } } - // Validate upload option + // Validate upload and download options stores := []string{"none", "s3", "sftp", "gcs", "azure"} if err := validateEnum(opts.Upload, stores); err != nil { return opts, changed, fmt.Errorf("invalid value for --upload: %s", err) } + if err := validateEnum(opts.Download, stores); err != nil { + return opts, changed, fmt.Errorf("invalid value for --download: %s", err) + } + opts.PurgeRemote, err = validateYesNoOption(*purgeRemote) if err != nil { return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err) } - switch opts.Upload { - case "s3": - // Validate S3 options - opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath) - if err != nil { - return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err) - } + for _, o := range []string{opts.Upload, opts.Download} { + switch o { + case "s3": + // Validate S3 options + opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath) + if err != nil { + return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err) + } - S3WithTLS, err := validateYesNoOption(*S3UseTLS) - if err != nil { - return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err) - } - opts.S3DisableTLS = !S3WithTLS + S3WithTLS, err := validateYesNoOption(*S3UseTLS) + if err != nil { + return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err) + } + opts.S3DisableTLS = !S3WithTLS - case "sftp": - opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey) - if err != nil { - return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err) + case "sftp": + opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey) + if err != nil { + return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err) + } } } @@ -816,6 +825,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin case "upload": opts.Upload = cliOpts.Upload + case "download": + opts.Download = cliOpts.Download case "purge-remote": opts.PurgeRemote = cliOpts.PurgeRemote diff --git a/config_test.go b/config_test.go index a627e84..49f556a 100644 --- a/config_test.go +++ b/config_test.go @@ -195,6 +195,7 @@ func TestDefaultOptions(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", } @@ -238,6 +239,7 @@ func TestParseCli(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -262,6 +264,7 @@ func TestParseCli(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -311,6 +314,7 @@ func TestParseCli(t *testing.T) { CipherPassphrase: "testpass", WithRolePasswords: true, Upload: "wrong", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -318,6 +322,32 @@ func TestParseCli(t *testing.T) { "invalid value for --upload: value not found in [none s3 sftp gcs azure]", "", }, + { + []string{"--download", "wrong"}, + options{ + Directory: "/var/backups/postgresql", + Format: 'c', + DirJobs: 1, + CompressLevel: -1, + Jobs: 1, + PauseTimeout: 3600, + PurgeInterval: -30 * 24 * time.Hour, + PurgeKeep: 0, + SumAlgo: "none", + CfgFile: "/etc/pg_back/pg_back.conf", + TimeFormat: timeFormat, + Encrypt: true, + CipherPassphrase: "testpass", + WithRolePasswords: true, + Upload: "none", + Download: "wrong", + AzureEndpoint: "blob.core.windows.net", + }, + false, + false, + "invalid value for --download: value not found in [none s3 sftp gcs azure]", + "", + }, { []string{"--decrypt", "--encrypt"}, defaults, @@ -344,6 +374,7 @@ func TestParseCli(t *testing.T) { CipherPassphrase: "mypass", WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -369,6 +400,7 @@ func TestParseCli(t *testing.T) { CipherPrivateKey: "mykey", WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -394,6 +426,7 @@ func TestParseCli(t *testing.T) { CipherPublicKey: "fakepubkey", WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, false, @@ -498,6 +531,7 @@ func TestLoadConfigurationFile(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -519,6 +553,7 @@ func TestLoadConfigurationFile(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -539,6 +574,7 @@ func TestLoadConfigurationFile(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -559,6 +595,7 @@ func TestLoadConfigurationFile(t *testing.T) { TimeFormat: "2006-01-02_15-04-05", WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -608,6 +645,7 @@ func TestLoadConfigurationFile(t *testing.T) { }}, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -649,6 +687,7 @@ func TestLoadConfigurationFile(t *testing.T) { }}, WithRolePasswords: false, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", }, }, @@ -715,6 +754,7 @@ func TestMergeCliAndConfigoptions(t *testing.T) { TimeFormat: timeFormat, WithRolePasswords: true, Upload: "none", + Download: "none", AzureEndpoint: "blob.core.windows.net", } diff --git a/main.go b/main.go index ce92073..8a7cd8f 100644 --- a/main.go +++ b/main.go @@ -173,9 +173,21 @@ func run() (retVal error) { return fmt.Errorf("required cipher parameters not present: %w", err) } - // When asked to decrypt the backups, do it here and exit, we have all + if (opts.Upload == "s3" || opts.Download == "s3") && opts.S3Bucket == "" { + return fmt.Errorf("a bucket is mandatory with s3") + } + + if (opts.Upload == "gcs" || opts.Download == "gcs") && opts.GCSBucket == "" { + return fmt.Errorf("a bucket is mandatory with gcs") + } + + if (opts.Upload == "azure" || opts.Download == "azure") && opts.AzureContainer == "" { + return fmt.Errorf("a container is mandatory with azure") + } + + // When asked to download or decrypt the backups, do it here and exit, we have all // required input (passphrase and backup directory) - if opts.Decrypt { + if opts.Decrypt || opts.Download != "none" { // Avoid getting wrong globs from the config file since we are // using the remaining args from the command line that are // usually as a list of databases to dump @@ -187,9 +199,17 @@ func run() (retVal error) { } } - params := decryptParams{PrivateKey: opts.CipherPrivateKey, Passphrase: opts.CipherPassphrase} - if err := decryptDirectory(opts.Directory, params, opts.Jobs, globs); err != nil { - return err + if opts.Download != "none" { + if err := downloadFiles(opts.Download, opts, opts.Directory, globs); err != nil { + return err + } + } + + if opts.Decrypt { + params := decryptParams{PrivateKey: opts.CipherPrivateKey, Passphrase: opts.CipherPassphrase} + if err := decryptDirectory(opts.Directory, params, opts.Jobs, globs); err != nil { + return err + } } return nil @@ -211,18 +231,6 @@ func run() (retVal error) { return fmt.Errorf("provided pg_dump is older than 8.4, unable use it.") } - if opts.Upload == "s3" && opts.S3Bucket == "" { - return fmt.Errorf("a bucket is mandatory when upload is s3") - } - - if opts.Upload == "gcs" && opts.GCSBucket == "" { - return fmt.Errorf("a bucket is mandatory when upload is gcs") - } - - if opts.Upload == "azure" && opts.AzureContainer == "" { - return fmt.Errorf("a container is mandatory when upload is azure") - } - // Parse the connection information l.Verboseln("processing input connection parameters") conninfo, err := prepareConnInfo(opts.Host, opts.Port, opts.Username, opts.ConnDb) @@ -996,6 +1004,60 @@ func dumpConfigFiles(dir string, timeFormat string, db *pg, fc chan<- sumFileJob return nil } +func downloadFiles(repoName string, opts options, dir string, globs []string) error { + repo, err := NewRepo(repoName, opts) + if err != nil { + return err + } + + // Without globs, there is nothing to download + if len(globs) == 0 { + return fmt.Errorf("no filter given to download files, use globs as command line arguments") + } + + remoteFiles, err := repo.List("") + if err != nil { + return fmt.Errorf("could not list contents of remote location: %w", err) + } + + for _, i := range remoteFiles { + keep := false + for _, glob := range globs { + keep, err = filepath.Match(glob, i.key) + if err != nil { + return fmt.Errorf("bad patern: %w", err) + } + + if keep { + break + } + } + + if !keep { + l.Verboseln("skipping:", i.key) + continue + } + + if i.isDir { + l.Warnf("%s is a directory, append %c* to the filter to download its contents", i.key, os.PathSeparator) + continue + } + + // Create any parent directory under target dir + path := filepath.Join(dir, i.key) + parent := filepath.Dir(path) + if err := os.MkdirAll(parent, 0700); err != nil { + return fmt.Errorf("could not create directory %s: %w", parent, err) + } + + if err := repo.Download(i.key, path); err != nil { + return err + } + } + + return nil +} + func decryptDirectory(dir string, params decryptParams, workers int, globs []string) error { // Run a pool of workers to decrypt concurrently @@ -1386,40 +1448,11 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options) }(i) } - var ( - repo Repo - err error - ) - - switch opts.Upload { - case "s3": - repo, err = NewS3Repo(opts) - if err != nil { - l.Errorln("failed to prepare upload to S3:", err) - ret <- err - repo = nil - } - case "sftp": - repo, err = NewSFTPRepo(opts) - if err != nil { - l.Errorln("failed to prepare upload over SFTP:", err) - ret <- err - repo = nil - } - case "gcs": - repo, err = NewGCSRepo(opts) - if err != nil { - l.Errorln("failed to prepare upload to GCS:", err) - ret <- err - repo = nil - } - case "azure": - repo, err = NewAzRepo(opts) - if err != nil { - l.Errorln("failed to prepare upload to Azure", err) - ret <- err - repo = nil - } + repo, err := NewRepo(opts.Upload, opts) + if err != nil { + l.Errorln(err) + ret <- err + repo = nil } for i := 0; i < opts.Jobs; i++ { diff --git a/upload.go b/upload.go index 5b48eeb..064c081 100644 --- a/upload.go +++ b/upload.go @@ -56,6 +56,9 @@ type Repo interface { // Upload a path to the remote naming it target Upload(path string, target string) error + // Download target from the remote and store it into path + Download(target string, path string) error + // List remote files starting with a prefix. the prefix can be empty to // list all files List(prefix string) ([]Item, error) @@ -78,6 +81,38 @@ func forwardSlashes(target string) string { return strings.ReplaceAll(target, fmt.Sprintf("%c", os.PathSeparator), "/") } +func NewRepo(kind string, opts options) (Repo, error) { + var ( + repo Repo + err error + ) + + switch kind { + case "s3": + repo, err = NewS3Repo(opts) + if err != nil { + return nil, fmt.Errorf("failed to prepare S3 repo: %w", err) + } + case "sftp": + repo, err = NewSFTPRepo(opts) + if err != nil { + return nil, fmt.Errorf("failed to prepare sftp repo: %w", err) + } + case "gcs": + repo, err = NewGCSRepo(opts) + if err != nil { + return nil, fmt.Errorf("failed to prepare CGS repo: %w", err) + } + case "azure": + repo, err = NewAzRepo(opts) + if err != nil { + return nil, fmt.Errorf("failed to prepare Azure repo: %w", err) + } + } + + return repo, nil +} + type s3repo struct { region string bucket string @@ -169,6 +204,28 @@ func (r *s3repo) Upload(path string, target string) error { return nil } +func (r *s3repo) Download(target string, path string) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("download error: %w", err) + } + defer file.Close() + + downloader := s3manager.NewDownloader(r.session) + + l.Infof("downloading %s from S3 bucket %s to %s\n", target, r.bucket, path) + _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(r.bucket), + Key: aws.String(forwardSlashes(target)), + }) + + if err != nil { + return fmt.Errorf("unable to download %q from %q: %w", target, r.bucket, err) + } + + return nil +} + func (r *s3repo) List(prefix string) ([]Item, error) { svc := s3.New(r.session) @@ -482,6 +539,36 @@ func (r *sftpRepo) Upload(path string, target string) error { return nil } +func (r *sftpRepo) Download(target string, path string) error { + l.Infof("downloading %s from %s:%s using sftp\n", target, r.host, r.baseDir) + + dst, err := os.Create(path) + if err != nil { + return fmt.Errorf("sftp: could not open or create %s: %w", path, err) + } + defer dst.Close() + + rpath := filepath.Join(r.baseDir, target) + + // sftp requires slash as path separator + if os.PathSeparator != '/' { + rpath = strings.ReplaceAll(rpath, string(os.PathSeparator), "/") + } + l.Verboseln("sftp remote path is:", rpath) + + src, err := r.client.Open(rpath) + if err != nil { + return fmt.Errorf("sftp: could not open %s on %s: %w", rpath, r.host, err) + } + defer src.Close() + + if _, err := io.Copy(dst, src); err != nil { + return fmt.Errorf("sftp: could not receive data with sftp: %s", err) + } + + return nil +} + func (r *sftpRepo) List(prefix string) (items []Item, rerr error) { items = make([]Item, 0) @@ -592,6 +679,27 @@ func (r *gcsRepo) Upload(path string, target string) error { return obj.Close() } +func (r *gcsRepo) Download(target string, path string) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("download error: %w", err) + } + defer file.Close() + + obj, err := r.client.Bucket(r.bucket).Object(forwardSlashes(target)).NewReader(context.Background()) + if err != nil { + return fmt.Errorf("download error: %w", err) + } + defer obj.Close() + + l.Infof("downloading %s from GCS bucket %s to %s\n", target, r.bucket, path) + if _, err := io.Copy(file, obj); err != nil { + return fmt.Errorf("could not read data from GCS object: %w", err) + } + + return obj.Close() +} + func (r *gcsRepo) List(prefix string) (items []Item, rerr error) { items = make([]Item, 0) @@ -694,6 +802,22 @@ func (r *azRepo) Upload(path string, target string) error { return nil } +func (r *azRepo) Download(target string, path string) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("download error: %w", err) + } + defer file.Close() + + l.Infof("downloading %s from Azure container %s\n", target, r.container) + _, err = r.client.DownloadFile(context.Background(), r.container, target, file, nil) + if err != nil { + return fmt.Errorf("could not download %s from Azure: %w", target, err) + } + + return nil +} + func (r *azRepo) List(prefix string) ([]Item, error) { p := forwardSlashes(prefix) pager := r.client.NewListBlobsFlatPager(r.container, &azblob.ListBlobsFlatOptions{