diff --git a/app/app.go b/app/app.go index 33f4620..592330f 100644 --- a/app/app.go +++ b/app/app.go @@ -70,6 +70,7 @@ const ( SERVER_PORT = "server:port" SERVER_ACCESS_TOKEN = "server:access-token" STORAGE_TYPE = "storage:type" + STORAGE_ENCRYPTION_KEY = "storage:encryption-key" STORAGE_FS_PATH = "storage-fs:path" STORAGE_FS_MODE = "storage-fs:mode" STORAGE_SFTP_HOST = "storage-sftp:host" @@ -338,6 +339,13 @@ func validateConfig() error { ) } + if knfu.GetS(STORAGE_ENCRYPTION_KEY) != "" { + validators = append(validators, + &knf.Validator{STORAGE_ENCRYPTION_KEY, knfv.LenGreater, 16}, + &knf.Validator{STORAGE_ENCRYPTION_KEY, knfv.LenLess, 96}, + ) + } + errs := knfu.Validate(validators) if len(errs) > 0 { diff --git a/app/basic.go b/app/basic.go index 60c42ce..f4b84d3 100644 --- a/app/basic.go +++ b/app/basic.go @@ -8,30 +8,21 @@ package app // ////////////////////////////////////////////////////////////////////////////////// // import ( - "encoding/base64" "fmt" - "os" - "time" "github.com/essentialkaos/ek/v13/events" "github.com/essentialkaos/ek/v13/fmtc" "github.com/essentialkaos/ek/v13/fmtutil" - "github.com/essentialkaos/ek/v13/fsutil" "github.com/essentialkaos/ek/v13/log" "github.com/essentialkaos/ek/v13/options" "github.com/essentialkaos/ek/v13/path" "github.com/essentialkaos/ek/v13/spinner" - "github.com/essentialkaos/ek/v13/timeutil" + "github.com/essentialkaos/ek/v13/terminal" knfu "github.com/essentialkaos/ek/v13/knf/united" "github.com/essentialkaos/atlassian-cloud-backuper/backuper" - "github.com/essentialkaos/atlassian-cloud-backuper/backuper/confluence" - "github.com/essentialkaos/atlassian-cloud-backuper/backuper/jira" "github.com/essentialkaos/atlassian-cloud-backuper/uploader" - "github.com/essentialkaos/atlassian-cloud-backuper/uploader/fs" - "github.com/essentialkaos/atlassian-cloud-backuper/uploader/s3" - "github.com/essentialkaos/atlassian-cloud-backuper/uploader/sftp" ) // ////////////////////////////////////////////////////////////////////////////////// // @@ -45,6 +36,12 @@ func startApp(args options.Arguments) error { addEventsHandlers(dispatcher) } + if knfu.GetS(STORAGE_ENCRYPTION_KEY) != "" { + fmtc.NewLine() + terminal.Warn("▲ Backup will be encrypted while uploading. You will not be able to use the") + terminal.Warn(" backup if you lose the encryption key. Keep it in a safe place.") + } + defer temp.Clean() target := args.Get(0).String() @@ -93,116 +90,6 @@ func startApp(args options.Arguments) error { return nil } -// getBackuper returns backuper instances -func getBackuper(target string) (backuper.Backuper, error) { - var err error - var bkpr backuper.Backuper - - bkpConfig, err := getBackuperConfig(target) - - if err != nil { - return nil, err - } - - switch target { - case TARGET_JIRA: - bkpr, err = jira.NewBackuper(bkpConfig) - case TARGET_CONFLUENCE: - bkpr, err = confluence.NewBackuper(bkpConfig) - } - - return bkpr, nil -} - -// getOutputFileName returns name for backup output file -func getOutputFileName(target string) string { - var template string - - switch target { - case TARGET_JIRA: - template = knfu.GetS(JIRA_OUTPUT_FILE, `jira-backup-%Y-%m-%d`) + ".zip" - case TARGET_CONFLUENCE: - template = knfu.GetS(JIRA_OUTPUT_FILE, `confluence-backup-%Y-%m-%d`) + ".zip" - } - - return timeutil.Format(time.Now(), template) -} - -// getBackuperConfig returns configuration for backuper -func getBackuperConfig(target string) (*backuper.Config, error) { - switch target { - case TARGET_JIRA: - return &backuper.Config{ - Account: knfu.GetS(ACCESS_ACCOUNT), - Email: knfu.GetS(ACCESS_EMAIL), - APIKey: knfu.GetS(ACCESS_API_KEY), - WithAttachments: knfu.GetB(JIRA_INCLUDE_ATTACHMENTS), - ForCloud: knfu.GetB(JIRA_CLOUD_FORMAT), - }, nil - - case TARGET_CONFLUENCE: - return &backuper.Config{ - Account: knfu.GetS(ACCESS_ACCOUNT), - Email: knfu.GetS(ACCESS_EMAIL), - APIKey: knfu.GetS(ACCESS_API_KEY), - WithAttachments: knfu.GetB(CONFLUENCE_INCLUDE_ATTACHMENTS), - ForCloud: knfu.GetB(CONFLUENCE_CLOUD_FORMAT), - }, nil - } - - return nil, fmt.Errorf("Unknown target %q", target) -} - -// getUploader returns uploader instance -func getUploader(target string) (uploader.Uploader, error) { - var err error - var updr uploader.Uploader - - switch knfu.GetS(STORAGE_TYPE) { - case STORAGE_FS: - updr, err = fs.NewUploader(&fs.Config{ - Path: path.Join(knfu.GetS(STORAGE_FS_PATH), target), - Mode: knfu.GetM(STORAGE_FS_MODE, 0600), - }) - - case STORAGE_SFTP: - keyData, err := readPrivateKeyData() - - if err != nil { - return nil, err - } - - updr, err = sftp.NewUploader(&sftp.Config{ - Host: knfu.GetS(STORAGE_SFTP_HOST), - User: knfu.GetS(STORAGE_SFTP_USER), - Key: keyData, - Path: path.Join(knfu.GetS(STORAGE_SFTP_PATH), target), - Mode: knfu.GetM(STORAGE_SFTP_MODE, 0600), - }) - - case STORAGE_S3: - updr, err = s3.NewUploader(&s3.Config{ - Host: knfu.GetS(STORAGE_S3_HOST), - Region: knfu.GetS(STORAGE_S3_REGION), - AccessKeyID: knfu.GetS(STORAGE_S3_ACCESS_KEY), - SecretKey: knfu.GetS(STORAGE_S3_SECRET_KEY), - Bucket: knfu.GetS(STORAGE_S3_BUCKET), - Path: path.Join(knfu.GetS(STORAGE_S3_PATH), target), - }) - } - - return updr, err -} - -// readPrivateKeyData reads private key data -func readPrivateKeyData() ([]byte, error) { - if fsutil.IsExist(knfu.GetS(STORAGE_SFTP_KEY)) { - return os.ReadFile(knfu.GetS(STORAGE_SFTP_KEY)) - } - - return base64.StdEncoding.DecodeString(knfu.GetS(STORAGE_SFTP_KEY)) -} - // addEventsHandlers registers events handlers func addEventsHandlers(dispatcher *events.Dispatcher) { dispatcher.AddHandler(backuper.EVENT_BACKUP_STARTED, func(payload any) { @@ -212,7 +99,7 @@ func addEventsHandlers(dispatcher *events.Dispatcher) { dispatcher.AddHandler(backuper.EVENT_BACKUP_PROGRESS, func(payload any) { p := payload.(*backuper.ProgressInfo) - spinner.Update("[%d%%] %s", p.Progress, p.Message) + spinner.Update("{s}(%d%%){!} %s", p.Progress, p.Message) }) dispatcher.AddHandler(backuper.EVENT_BACKUP_SAVING, func(payload any) { @@ -231,7 +118,7 @@ func addEventsHandlers(dispatcher *events.Dispatcher) { dispatcher.AddHandler(uploader.EVENT_UPLOAD_PROGRESS, func(payload any) { p := payload.(*uploader.ProgressInfo) spinner.Update( - "[%s] Uploading file (%s/%s)", + "{s}(%5s){!} Uploading file {s-}(%7s | %7s){!}", fmtutil.PrettyPerc(p.Progress), fmtutil.PrettySize(p.Current), fmtutil.PrettySize(p.Total), diff --git a/app/common.go b/app/common.go new file mode 100644 index 0000000..2aec48c --- /dev/null +++ b/app/common.go @@ -0,0 +1,151 @@ +package app + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2024 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "encoding/base64" + "fmt" + "os" + "time" + + "github.com/essentialkaos/ek/v13/fsutil" + "github.com/essentialkaos/ek/v13/path" + "github.com/essentialkaos/ek/v13/timeutil" + + "github.com/essentialkaos/katana" + + knfu "github.com/essentialkaos/ek/v13/knf/united" + + "github.com/essentialkaos/atlassian-cloud-backuper/backuper" + "github.com/essentialkaos/atlassian-cloud-backuper/backuper/confluence" + "github.com/essentialkaos/atlassian-cloud-backuper/backuper/jira" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader/fs" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader/s3" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader/sftp" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// getBackuper returns backuper instances +func getBackuper(target string) (backuper.Backuper, error) { + var err error + var bkpr backuper.Backuper + + bkpConfig, err := getBackuperConfig(target) + + if err != nil { + return nil, err + } + + switch target { + case TARGET_JIRA: + bkpr, err = jira.NewBackuper(bkpConfig) + case TARGET_CONFLUENCE: + bkpr, err = confluence.NewBackuper(bkpConfig) + } + + return bkpr, nil +} + +// getOutputFileName returns name for backup output file +func getOutputFileName(target string) string { + var template string + + switch target { + case TARGET_JIRA: + template = knfu.GetS(JIRA_OUTPUT_FILE, `jira-backup-%Y-%m-%d`) + ".zip" + case TARGET_CONFLUENCE: + template = knfu.GetS(JIRA_OUTPUT_FILE, `confluence-backup-%Y-%m-%d`) + ".zip" + } + + return timeutil.Format(time.Now(), template) +} + +// getBackuperConfig returns configuration for backuper +func getBackuperConfig(target string) (*backuper.Config, error) { + switch target { + case TARGET_JIRA: + return &backuper.Config{ + Account: knfu.GetS(ACCESS_ACCOUNT), + Email: knfu.GetS(ACCESS_EMAIL), + APIKey: knfu.GetS(ACCESS_API_KEY), + WithAttachments: knfu.GetB(JIRA_INCLUDE_ATTACHMENTS), + ForCloud: knfu.GetB(JIRA_CLOUD_FORMAT), + }, nil + + case TARGET_CONFLUENCE: + return &backuper.Config{ + Account: knfu.GetS(ACCESS_ACCOUNT), + Email: knfu.GetS(ACCESS_EMAIL), + APIKey: knfu.GetS(ACCESS_API_KEY), + WithAttachments: knfu.GetB(CONFLUENCE_INCLUDE_ATTACHMENTS), + ForCloud: knfu.GetB(CONFLUENCE_CLOUD_FORMAT), + }, nil + } + + return nil, fmt.Errorf("Unknown target %q", target) +} + +// getUploader returns uploader instance +func getUploader(target string) (uploader.Uploader, error) { + var err error + var updr uploader.Uploader + var secret *katana.Secret + + if knfu.GetS(STORAGE_ENCRYPTION_KEY) != "" { + secret = katana.NewSecret(knfu.GetS(STORAGE_ENCRYPTION_KEY)) + } + + switch knfu.GetS(STORAGE_TYPE) { + case STORAGE_FS: + updr, err = fs.NewUploader(&fs.Config{ + Path: path.Join(knfu.GetS(STORAGE_FS_PATH), target), + Mode: knfu.GetM(STORAGE_FS_MODE, 0600), + Secret: secret, + }) + + case STORAGE_SFTP: + keyData, err := readPrivateKeyData() + + if err != nil { + return nil, err + } + + updr, err = sftp.NewUploader(&sftp.Config{ + Host: knfu.GetS(STORAGE_SFTP_HOST), + User: knfu.GetS(STORAGE_SFTP_USER), + Key: keyData, + Path: path.Join(knfu.GetS(STORAGE_SFTP_PATH), target), + Mode: knfu.GetM(STORAGE_SFTP_MODE, 0600), + Secret: secret, + }) + + case STORAGE_S3: + updr, err = s3.NewUploader(&s3.Config{ + Host: knfu.GetS(STORAGE_S3_HOST), + Region: knfu.GetS(STORAGE_S3_REGION), + AccessKeyID: knfu.GetS(STORAGE_S3_ACCESS_KEY), + SecretKey: knfu.GetS(STORAGE_S3_SECRET_KEY), + Bucket: knfu.GetS(STORAGE_S3_BUCKET), + Path: path.Join(knfu.GetS(STORAGE_S3_PATH), target), + Secret: secret, + }) + } + + return updr, err +} + +// readPrivateKeyData reads private key data +func readPrivateKeyData() ([]byte, error) { + if fsutil.IsExist(knfu.GetS(STORAGE_SFTP_KEY)) { + return os.ReadFile(knfu.GetS(STORAGE_SFTP_KEY)) + } + + return base64.StdEncoding.DecodeString(knfu.GetS(STORAGE_SFTP_KEY)) +} diff --git a/app/server.go b/app/server.go index 6ba5670..8d25121 100644 --- a/app/server.go +++ b/app/server.go @@ -150,7 +150,7 @@ func downloadBackupHandler(rw http.ResponseWriter, r *http.Request) { log.Info("Uploading backup to storage", lf) - err = updr.Write(br, outputFile) + err = updr.Write(br, outputFile, 0) if err != nil { log.Error("Can't upload backup file: %v", err, lf) diff --git a/backuper/confluence/confluence-backuper.go b/backuper/confluence/confluence-backuper.go index b549f1e..e445c70 100644 --- a/backuper/confluence/confluence-backuper.go +++ b/backuper/confluence/confluence-backuper.go @@ -98,8 +98,17 @@ func (b *ConfluenceBackuper) Start() (string, error) { info, _ := b.getBackupProgress() if info != nil && !info.IsOutdated { - log.Info("Found previously created backup task") + log.Info( + "Found previously created backup task", + log.F{"backup-status", info.CurrentStatus}, + log.F{"backup-perc", info.AlternativePercentage}, + log.F{"backup-size", info.Size}, + log.F{"backup-file", info.Filename}, + log.F{"backup-outdated", info.IsOutdated}, + ) } else { + log.Info("No previously created backup task or task is outdated, starting new backup…") + err := b.startBackup() if err != nil { @@ -142,14 +151,14 @@ func (b *ConfluenceBackuper) Progress(taskID string) (string, error) { if progressInfo.Size == 0 && progressInfo.AlternativePercentage >= lastProgress { log.Info( - "(%s) Backup in progress: %s", + "(%s%%) Backup in progress: %s", progressInfo.AlternativePercentage, progressInfo.CurrentStatus, ) lastProgress = progressInfo.AlternativePercentage } - if progressInfo.Size != 0 && progressInfo.Filename != "" { + if progressInfo.Filename != "" { backupFileURL = progressInfo.Filename break } diff --git a/backuper/jira/jira-backuper.go b/backuper/jira/jira-backuper.go index c5afc0f..a189d56 100644 --- a/backuper/jira/jira-backuper.go +++ b/backuper/jira/jira-backuper.go @@ -104,7 +104,7 @@ func (b *JiraBackuper) Start() (string, error) { if backupTaskID != "" { log.Info("Found previously created backup task with ID %s", backupTaskID) } else { - log.Info("No previously created task found, run backup…") + log.Info("No previously created task found, starting new backup…") backupTaskID, err = b.startBackup() diff --git a/common/atlassian-cloud-backuper.knf b/common/atlassian-cloud-backuper.knf index 4cf38fd..fa40a4a 100644 --- a/common/atlassian-cloud-backuper.knf +++ b/common/atlassian-cloud-backuper.knf @@ -25,6 +25,9 @@ # Storage type (fs/sftp/s3) type: + # Katana encryption key + encryption-key: + [storage-fs] # Path to directory with backups diff --git a/go.mod b/go.mod index 3abd02a..7043e3f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/credentials v1.17.27 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 - github.com/essentialkaos/ek/v13 v13.1.0 + github.com/essentialkaos/ek/v13 v13.2.0 + github.com/essentialkaos/katana v0.2.0 github.com/pkg/sftp v1.13.6 golang.org/x/crypto v0.25.0 ) @@ -22,6 +23,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect github.com/aws/smithy-go v1.20.3 // indirect github.com/essentialkaos/depsy v1.3.0 // indirect + github.com/essentialkaos/sio v1.0.0 // indirect github.com/kr/fs v0.1.0 // indirect golang.org/x/sys v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 6890c11..b04bb0d 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,12 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/essentialkaos/check v1.4.0 h1:kWdFxu9odCxUqo1NNFNJmguGrDHgwi3A8daXX1nkuKk= github.com/essentialkaos/depsy v1.3.0 h1:CN7bRgBU2jGTHSkg/Sh38eDUn7cvmaTp2sxFt2HpFeU= github.com/essentialkaos/depsy v1.3.0/go.mod h1:kpiTAV17dyByVnrbNaMcZt2jRwvuXClUYOzpyJQwtG8= -github.com/essentialkaos/ek/v13 v13.1.0 h1:k0X7805R2z5QVBCYzGgApWosnpmBdb+00B4rc2tEGVA= -github.com/essentialkaos/ek/v13 v13.1.0/go.mod h1:RVf1NpNyK04xkBJ3NTUD1wNLWemY9/naVD4iEVjU2fA= +github.com/essentialkaos/ek/v13 v13.2.0 h1:Ra6segoyFYjtdz5eh0mQxJMeIso7h61A7IyG9B4R6bI= +github.com/essentialkaos/ek/v13 v13.2.0/go.mod h1:RVf1NpNyK04xkBJ3NTUD1wNLWemY9/naVD4iEVjU2fA= +github.com/essentialkaos/katana v0.2.0 h1:LRnKyEHFET9P45L718DI704oUBHcOjW+/bWBstPb9qg= +github.com/essentialkaos/katana v0.2.0/go.mod h1:B0IUikFvR6Iutx93iSu3xezHfHvIuIgXJSO6Agujp+0= +github.com/essentialkaos/sio v1.0.0 h1:+VZg0Z7+Cx8F/FmlczzTJYM6rq/LhTR45Rsditmu0Ec= +github.com/essentialkaos/sio v1.0.0/go.mod h1:lKaW6IPMJ8GAEAiXe175zcEld370u3Nr546c22Kw5C8= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/uploader/fs/fs.go b/uploader/fs/fs.go index 12bab41..f9166b2 100644 --- a/uploader/fs/fs.go +++ b/uploader/fs/fs.go @@ -12,12 +12,16 @@ import ( "fmt" "io" "os" + "time" "github.com/essentialkaos/ek/v13/events" "github.com/essentialkaos/ek/v13/fsutil" "github.com/essentialkaos/ek/v13/log" + "github.com/essentialkaos/ek/v13/passthru" "github.com/essentialkaos/ek/v13/path" + "github.com/essentialkaos/katana" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader" ) @@ -25,8 +29,9 @@ import ( // Config is configuration for FS uploader type Config struct { - Path string - Mode os.FileMode + Path string + Mode os.FileMode + Secret *katana.Secret } // FSUploader is FS uploader instance @@ -64,10 +69,6 @@ func (u *FSUploader) SetDispatcher(d *events.Dispatcher) { // Upload uploads given file to storage func (u *FSUploader) Upload(file, fileName string) error { - log.Info("Copying backup file to %s…", u.config.Path) - - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "FS") - err := fsutil.ValidatePerms("FRS", file) if err != nil { @@ -78,43 +79,89 @@ func (u *FSUploader) Upload(file, fileName string) error { err = os.MkdirAll(u.config.Path, 0750) if err != nil { - return fmt.Errorf("Can't create directory for backup: %v", err) + return fmt.Errorf("Can't create directory for backup: %w", err) } } - err = fsutil.CopyFile(file, path.Join(u.config.Path, fileName), u.config.Mode) + fd, err := os.Open(file) - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_DONE, "FS") + if err != nil { + return fmt.Errorf("Can't open backup file: %w", err) + } - log.Info("Backup successfully copied to %s", u.config.Path) + defer fd.Close() + + err = u.Write(fd, fileName, fsutil.GetSize(file)) + + if err != nil { + return fmt.Errorf("Can't save backup file: %w", err) + } return err } // Write writes data from given reader to given file -func (u *FSUploader) Write(r io.ReadCloser, fileName string) error { +func (u *FSUploader) Write(r io.ReadCloser, fileName string, fileSize int64) error { u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "FS") - fd, err := os.OpenFile( - path.Join(u.config.Path, fileName), - os.O_CREATE|os.O_TRUNC|os.O_WRONLY, u.config.Mode, - ) + var w io.Writer + + lastUpdate := time.Now() + outputFile := path.Join(u.config.Path, fileName) + + log.Info("Copying backup file to %s…", u.config.Path) + + fd, err := os.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, u.config.Mode) if err != nil { return err } - defer fd.Close() - defer r.Close() + w = fd + + if u.config.Secret != nil { + sw, err := u.config.Secret.NewWriter(fd) + + if err != nil { + return fmt.Errorf("Can't create encrypted writer: %w", err) + } + + defer sw.Close() + + w = sw + } - w := bufio.NewWriter(fd) - _, err = io.Copy(w, r) + if fileSize > 0 { + pw := passthru.NewWriter(w, fileSize) + + pw.Update = func(n int) { + if time.Since(lastUpdate) < 3*time.Second { + return + } + + u.dispatcher.Dispatch( + uploader.EVENT_UPLOAD_PROGRESS, + &uploader.ProgressInfo{ + Progress: pw.Progress(), + Current: pw.Current(), + Total: pw.Total(), + }, + ) + + lastUpdate = time.Now() + } + + w = pw + } + + _, err = io.Copy(bufio.NewWriter(w), r) if err != nil { return fmt.Errorf("File writing error: %w", err) } u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_DONE, "FS") + log.Info("Backup successfully copied to %s", u.config.Path) return nil } diff --git a/uploader/s3/s3.go b/uploader/s3/s3.go index c00b112..669682b 100644 --- a/uploader/s3/s3.go +++ b/uploader/s3/s3.go @@ -25,6 +25,8 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/essentialkaos/katana" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader" ) @@ -38,6 +40,7 @@ type Config struct { SecretKey string Bucket string Path string + Secret *katana.Secret } // S3Uploader is S3 uploader instance @@ -75,84 +78,34 @@ func (u *S3Uploader) SetDispatcher(d *events.Dispatcher) { // Upload uploads given file to S3 storage func (u *S3Uploader) Upload(file, fileName string) error { - var outputFile string - - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "S3") - - lastUpdate := time.Now() - fileSize := fsutil.GetSize(file) - - if u.config.Path == "" { - outputFile = fileName - } else { - outputFile = path.Join(u.config.Path, fileName) - } - - log.Info( - "Uploading backup file to %s:%s (%s/%s)", - u.config.Bucket, u.config.Path, u.config.Host, u.config.Region, - ) - - client := s3.New(s3.Options{ - Region: u.config.Region, - BaseEndpoint: aws.String("https://" + u.config.Host), - Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider( - u.config.AccessKeyID, u.config.SecretKey, "", - )), - }) - - inputFD, err := os.OpenFile(file, os.O_RDONLY, 0) + fd, err := os.Open(file) if err != nil { return fmt.Errorf("Can't open backup file for reading: %v", err) } - defer inputFD.Close() - - r := passthru.NewReader(inputFD, fileSize) + defer fd.Close() - r.Update = func(n int) { - if time.Since(lastUpdate) < 3*time.Second { - return - } - - u.dispatcher.Dispatch( - uploader.EVENT_UPLOAD_PROGRESS, - &uploader.ProgressInfo{ - Progress: r.Progress(), - Current: r.Current(), - Total: r.Total(), - }, - ) - - lastUpdate = time.Now() - } - - _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{ - Bucket: aws.String(u.config.Bucket), - Key: aws.String(outputFile), - Body: r, - }) + err = u.Write(fd, fileName, fsutil.GetSize(file)) if err != nil { - return fmt.Errorf("Can't upload file to S3: %v", err) + return fmt.Errorf("Can't save backup: %w", err) } - log.Info("File successfully uploaded to S3!") - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_DONE, "S3") - return nil } // Write writes data from given reader to given file -func (u *S3Uploader) Write(r io.ReadCloser, fileName string) error { - var outputFile string - +func (u *S3Uploader) Write(r io.ReadCloser, fileName string, fileSize int64) error { u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "S3") - if u.config.Path == "" { - outputFile = fileName - } else { + var rr io.Reader + var err error + + lastUpdate := time.Now() + outputFile := fileName + + if u.config.Path != "" { outputFile = path.Join(u.config.Path, fileName) } @@ -161,6 +114,39 @@ func (u *S3Uploader) Write(r io.ReadCloser, fileName string) error { u.config.Bucket, u.config.Path, u.config.Host, u.config.Region, ) + rr = r + + if u.config.Secret != nil { + sr, err := u.config.Secret.NewReader(r, katana.MODE_ENCRYPT) + + if err != nil { + return fmt.Errorf("Can't create encrypted reader: %w", err) + } + + rr = sr + } + + if fileSize > 0 { + pr := passthru.NewReader(rr, fileSize) + + pr.Update = func(n int) { + if time.Since(lastUpdate) < 3*time.Second { + return + } + + u.dispatcher.Dispatch( + uploader.EVENT_UPLOAD_PROGRESS, + &uploader.ProgressInfo{ + Progress: pr.Progress(), + Current: pr.Current(), + Total: pr.Total(), + }, + ) + } + + rr = pr + } + client := s3.New(s3.Options{ Region: u.config.Region, BaseEndpoint: aws.String("https://" + u.config.Host), @@ -169,10 +155,10 @@ func (u *S3Uploader) Write(r io.ReadCloser, fileName string) error { )), }) - _, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(u.config.Bucket), Key: aws.String(outputFile), - Body: r, + Body: rr, }) if err != nil { diff --git a/uploader/sftp/sftp.go b/uploader/sftp/sftp.go index 2e19dbb..a1e30c2 100644 --- a/uploader/sftp/sftp.go +++ b/uploader/sftp/sftp.go @@ -14,15 +14,17 @@ import ( "strings" "time" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "github.com/essentialkaos/ek/v13/events" "github.com/essentialkaos/ek/v13/fsutil" "github.com/essentialkaos/ek/v13/log" "github.com/essentialkaos/ek/v13/passthru" "github.com/essentialkaos/ek/v13/path" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + + "github.com/essentialkaos/katana" + "github.com/essentialkaos/atlassian-cloud-backuper/uploader" ) @@ -30,11 +32,12 @@ import ( // Config is configuration for SFTP uploader type Config struct { - Host string - User string - Key []byte - Path string - Mode os.FileMode + Host string + User string + Key []byte + Path string + Mode os.FileMode + Secret *katana.Secret } // SFTPUploader is SFTP uploader instance @@ -72,88 +75,30 @@ func (u *SFTPUploader) SetDispatcher(d *events.Dispatcher) { // Upload uploads given file to SFTP storage func (u *SFTPUploader) Upload(file, fileName string) error { - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "SFTP") - - lastUpdate := time.Now() - fileSize := fsutil.GetSize(file) - outputFile := path.Join(u.config.Path, fileName) - - log.Info( - "Uploading backup file to %s@%s~%s/%s…", - u.config.User, u.config.Host, u.config.Path, fileName, - ) - - sftpClient, err := u.connectToSFTP() - - if err != nil { - return fmt.Errorf("Can't connect to SFTP: %v", err) - } - - defer sftpClient.Close() - - _, err = sftpClient.Stat(u.config.Path) - - if err != nil { - err = sftpClient.MkdirAll(u.config.Path) - - if err != nil { - return fmt.Errorf("Can't create directory for backup: %v", err) - } - } - - outputFD, err := sftpClient.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY) - - if err != nil { - return fmt.Errorf("Can't create file of SFTP: %v", err) - } - - defer outputFD.Close() - - inputFD, err := os.OpenFile(file, os.O_RDONLY, 0) + fd, err := os.Open(file) if err != nil { return fmt.Errorf("Can't open backup file for reading: %v", err) } - defer inputFD.Close() - - w := passthru.NewWriter(outputFD, fileSize) - - w.Update = func(n int) { - if time.Since(lastUpdate) < 3*time.Second { - return - } - - u.dispatcher.Dispatch( - uploader.EVENT_UPLOAD_PROGRESS, - &uploader.ProgressInfo{Progress: w.Progress(), Current: w.Current(), Total: w.Total()}, - ) - - lastUpdate = time.Now() - } - - _, err = io.Copy(w, inputFD) + defer fd.Close() - if err != nil { - return fmt.Errorf("Can't upload file to SFTP: %v", err) - } - - err = sftpClient.Chmod(outputFile, u.config.Mode) + err = u.Write(fd, fileName, fsutil.GetSize(file)) if err != nil { - log.Error("Can't change file mode for uploaded file: %v", err) + return fmt.Errorf("Can't save backup: %w", err) } - log.Info("File successfully uploaded to SFTP!") - u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_DONE, "SFTP") - return nil } // Write writes data from given reader to given file -func (u *SFTPUploader) Write(r io.ReadCloser, fileName string) error { +func (u *SFTPUploader) Write(r io.ReadCloser, fileName string, fileSize int64) error { u.dispatcher.DispatchAndWait(uploader.EVENT_UPLOAD_STARTED, "SFTP") + var w io.Writer + + lastUpdate := time.Now() outputFile := path.Join(u.config.Path, fileName) log.Info( @@ -179,16 +124,50 @@ func (u *SFTPUploader) Write(r io.ReadCloser, fileName string) error { } } - outputFD, err := sftpClient.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY) + fd, err := sftpClient.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY) if err != nil { return fmt.Errorf("Can't create file of SFTP: %v", err) } - defer outputFD.Close() - defer r.Close() + w = fd + + if u.config.Secret != nil { + sw, err := u.config.Secret.NewWriter(fd) + + if err != nil { + return fmt.Errorf("Can't create encrypted writer: %w", err) + } + + defer sw.Close() + + w = sw + } + + if fileSize > 0 { + pw := passthru.NewWriter(w, fileSize) + + pw.Update = func(n int) { + if time.Since(lastUpdate) < 3*time.Second { + return + } + + u.dispatcher.Dispatch( + uploader.EVENT_UPLOAD_PROGRESS, + &uploader.ProgressInfo{ + Progress: pw.Progress(), + Current: pw.Current(), + Total: pw.Total(), + }, + ) + + lastUpdate = time.Now() + } + + w = pw + } - _, err = io.Copy(outputFD, r) + _, err = io.Copy(w, r) if err != nil { return fmt.Errorf("Can't upload file to SFTP: %v", err) diff --git a/uploader/uploader.go b/uploader/uploader.go index 76da9cf..92de1f6 100644 --- a/uploader/uploader.go +++ b/uploader/uploader.go @@ -33,12 +33,14 @@ type ProgressInfo struct { // Uploader is generic uploader interface type Uploader interface { - // Upload uploads given file to storage - Upload(file, fileName string) error - // SetDispatcher sets events dispatcher SetDispatcher(d *events.Dispatcher) + // Upload uploads given file to storage + Upload(file, fileName string) error + // Write writes data from given reader to given file - Write(r io.ReadCloser, fileName string) error + Write(r io.ReadCloser, fileName string, fileSize int64) error } + +// ////////////////////////////////////////////////////////////////////////////////// //