From b235e691d603f8afd18547633a3afc24697fbabf Mon Sep 17 00:00:00 2001 From: Leslie Leung Date: Wed, 8 Nov 2023 14:45:09 +0800 Subject: [PATCH] feat: support S3 as storage backend --- README.md | 2 +- README_zh.md | 2 +- cmd/rip/rip.go | 9 ++--- cmd/run/run.go | 4 +-- config/example.config.yaml | 10 +++++- go.mod | 10 ++++++ go.sum | 24 ++++++++++++++ internal/config/config.go | 13 ++++++-- internal/rip/rip.go | 18 +++++++--- internal/storage/s3.go | 66 +++++++++++++++++++++++++++++++++++++ internal/storage/storage.go | 5 +++ 11 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 internal/storage/s3.go diff --git a/README.md b/README.md index f457511..758f46b 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ For configuration, you can checkout this [example](config/example.config.yaml). REAPER supports multiple storage types. - [x] File -- [ ] AWS S3 +- [x] AWS S3 ## Run as docker container diff --git a/README_zh.md b/README_zh.md index 7a932b1..3a992d4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -74,7 +74,7 @@ reaper run REAPER支持多种存储类型。 - [x] 文件 -- [ ] AWS S3 +- [x] AWS S3 ## 使用 Docker 运行 diff --git a/cmd/rip/rip.go b/cmd/rip/rip.go index c623d61..5b44124 100644 --- a/cmd/rip/rip.go +++ b/cmd/rip/rip.go @@ -19,7 +19,7 @@ func runRip(cmd *cobra.Command, args []string) { // find repo in config cfg := config.GetIns() var repo config.Repository - storages := make([]config.Storage, 0) + storages := make([]config.MultiStorage, 0) var found bool for _, repository := range cfg.Repository { if repository.Name == repoName { @@ -34,18 +34,15 @@ func runRip(cmd *cobra.Command, args []string) { if repo.URL == "" { ui.ErrorfExit("Repository %s has no URL", repoName) } - found = false for _, storage := range repo.Storage { for _, s := range cfg.Storage { if s.Name == storage { storages = append(storages, s) - found = true - break } } } - if !found { - ui.ErrorfExit("Storage %s not found in config", repo.Storage) + if len(storages) != len(repo.Storage) { + ui.ErrorfExit("Storage missing in config") } if err := rip.Rip(repo, storages); err != nil { diff --git a/cmd/run/run.go b/cmd/run/run.go index 9145cc4..b1270b5 100644 --- a/cmd/run/run.go +++ b/cmd/run/run.go @@ -16,13 +16,13 @@ var Cmd = &cobra.Command{ func runRun(cmd *cobra.Command, args []string) { cfg := config.GetIns() - storageMap := make(map[string]config.Storage) + storageMap := make(map[string]config.MultiStorage) for _, storage := range cfg.Storage { storageMap[storage.Name] = storage } for _, repo := range cfg.Repository { - storages := make([]config.Storage, 0) + storages := make([]config.MultiStorage, 0) for _, storage := range repo.Storage { if s, ok := storageMap[storage]; !ok { ui.Errorf("Storage %s not found in config", storage) diff --git a/config/example.config.yaml b/config/example.config.yaml index f1479d5..7afd269 100644 --- a/config/example.config.yaml +++ b/config/example.config.yaml @@ -3,8 +3,16 @@ repository: url: github.com/leslieleung/reaper storage: - localFile + - blackblaze storage: - name: localFile type: file - path: ./repo \ No newline at end of file + path: ./repo + - name: blackblaze + type: s3 + endpoint: s3.us-west-000.backblazeb2.com + region: us-west-000 + bucket: your-bucket-name + accessKeyID: your-access-key-id + secretAccessKey: your-secret-access-key \ No newline at end of file diff --git a/go.mod b/go.mod index 8b53576..05387e8 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/connesc/cipherio v0.2.1 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dsnet/compress v0.0.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -35,18 +36,27 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.63 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.10.0 // indirect diff --git a/go.sum b/go.sum index cb47286..2fc7130 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -152,6 +154,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -188,6 +191,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -196,7 +201,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -213,8 +222,19 @@ github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -235,6 +255,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= @@ -244,6 +266,8 @@ github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWR github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= diff --git a/internal/config/config.go b/internal/config/config.go index 7355307..44346d7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,8 +6,8 @@ import ( ) type Config struct { - Repository []Repository `yaml:"repository"` - Storage []Storage `yaml:"storage"` + Repository []Repository `yaml:"repository"` + Storage []MultiStorage `yaml:"storage"` } type Repository struct { @@ -23,6 +23,15 @@ type Storage struct { Path string `yaml:"path"` } +type MultiStorage struct { + Storage `mapstructure:",squash"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + AccessKeyID string `yaml:"accessKeyID"` + SecretAccessKey string `yaml:"secretAccessKey"` +} + var Path string var vp *viper.Viper diff --git a/internal/rip/rip.go b/internal/rip/rip.go index dec5a52..907533a 100644 --- a/internal/rip/rip.go +++ b/internal/rip/rip.go @@ -15,7 +15,7 @@ import ( "time" ) -func Rip(repo config.Repository, storages []config.Storage) error { +func Rip(repo config.Repository, storages []config.MultiStorage) error { id := uuid.New().String() // get current directory currentDir, _ := os.Getwd() @@ -68,16 +68,24 @@ func Rip(repo config.Repository, storages []config.Storage) error { // handle storages for _, s := range storages { + var err error switch s.Type { - case "file": + case storage.FileStorage: fileBackend := storage.File{} - err := fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes()) + err = fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes()) + case storage.S3Storage: + s3Backend, err := storage.New(s.Endpoint, s.Bucket, s.Region, s.AccessKeyID, s.SecretAccessKey) if err != nil { - ui.Errorf("Error storing file, %s", err) + ui.Errorf("Error creating S3 backend, %s", err) return err } - ui.Printf("File %s stored", path.Join(s.Path, base)) + err = s3Backend.PutObject(base, archive.Bytes()) } + if err != nil { + ui.Errorf("Error storing file, %s", err) + return err + } + ui.Printf("File %s stored", path.Join(s.Path, base)) } // cleanup diff --git a/internal/storage/s3.go b/internal/storage/s3.go new file mode 100644 index 0000000..4014238 --- /dev/null +++ b/internal/storage/s3.go @@ -0,0 +1,66 @@ +package storage + +import ( + "bytes" + "context" + "github.com/leslieleung/reaper/internal/ui" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var _ Storage = (*S3)(nil) + +type S3 struct { + Endpoint string + Bucket string + Region string + AccessKeyID string + SecretAccessKey string + + client *minio.Client +} + +func New(endpoint, bucket, region, accessKeyID, secretAccessKey string) (*S3, error) { + s3 := &S3{ + Endpoint: endpoint, + Bucket: bucket, + Region: region, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + } + + client, err := minio.New(s3.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(s3.AccessKeyID, s3.SecretAccessKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + s3.client = client + return s3, nil +} + +func (s S3) ListObject(prefix string) ([]Object, error) { + // TODO implement me + panic("implement me") +} + +func (s S3) GetObject(identifier string) (Object, error) { + // TODO implement me + panic("implement me") +} + +func (s S3) PutObject(identifier string, data []byte) error { + size := int64(len(data)) + info, err := s.client.PutObject(context.Background(), s.Bucket, identifier, bytes.NewReader(data), size, minio.PutObjectOptions{}) + if err != nil { + return err + } + ui.Printf("info: %v", info) + return nil +} + +func (s S3) DeleteObject(identifier string) error { + // TODO implement me + panic("implement me") +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 1e4d44f..ae91d5a 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -2,6 +2,11 @@ package storage import "time" +const ( + FileStorage = "file" + S3Storage = "s3" +) + type Object struct { Path string Content []byte