Skip to content

Commit

Permalink
feat: support S3 as storage backend
Browse files Browse the repository at this point in the history
  • Loading branch information
LeslieLeung committed Nov 8, 2023
1 parent a0fdcf0 commit b235e69
Show file tree
Hide file tree
Showing 11 changed files with 145 additions and 18 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ For configuration, you can checkout this [example](config/example.config.yaml).
REAPER supports multiple storage types.

- [x] File
- [ ] AWS S3
- [x] AWS S3

## Run as docker container

Expand Down
2 changes: 1 addition & 1 deletion README_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ reaper run
REAPER支持多种存储类型。

- [x] 文件
- [ ] AWS S3
- [x] AWS S3

## 使用 Docker 运行

Expand Down
9 changes: 3 additions & 6 deletions cmd/rip/rip.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func runRip(cmd *cobra.Command, args []string) {
// find repo in config
cfg := config.GetIns()
var repo config.Repository
storages := make([]config.Storage, 0)
storages := make([]config.MultiStorage, 0)
var found bool
for _, repository := range cfg.Repository {
if repository.Name == repoName {
Expand All @@ -34,18 +34,15 @@ func runRip(cmd *cobra.Command, args []string) {
if repo.URL == "" {
ui.ErrorfExit("Repository %s has no URL", repoName)
}
found = false
for _, storage := range repo.Storage {
for _, s := range cfg.Storage {
if s.Name == storage {
storages = append(storages, s)
found = true
break
}
}
}
if !found {
ui.ErrorfExit("Storage %s not found in config", repo.Storage)
if len(storages) != len(repo.Storage) {
ui.ErrorfExit("Storage missing in config")
}

if err := rip.Rip(repo, storages); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ var Cmd = &cobra.Command{
func runRun(cmd *cobra.Command, args []string) {
cfg := config.GetIns()

storageMap := make(map[string]config.Storage)
storageMap := make(map[string]config.MultiStorage)
for _, storage := range cfg.Storage {
storageMap[storage.Name] = storage
}

for _, repo := range cfg.Repository {
storages := make([]config.Storage, 0)
storages := make([]config.MultiStorage, 0)
for _, storage := range repo.Storage {
if s, ok := storageMap[storage]; !ok {
ui.Errorf("Storage %s not found in config", storage)
Expand Down
10 changes: 9 additions & 1 deletion config/example.config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,16 @@ repository:
url: github.com/leslieleung/reaper
storage:
- localFile
- blackblaze

storage:
- name: localFile
type: file
path: ./repo
path: ./repo
- name: blackblaze
type: s3
endpoint: s3.us-west-000.backblazeb2.com
region: us-west-000
bucket: your-bucket-name
accessKeyID: your-access-key-id
secretAccessKey: your-secret-access-key
10 changes: 10 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/connesc/cipherio v0.2.1 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
Expand All @@ -35,18 +36,27 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.63 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
Expand Down
24 changes: 24 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
Expand Down Expand Up @@ -152,6 +154,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
Expand Down Expand Up @@ -188,6 +191,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
Expand All @@ -196,7 +201,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
Expand All @@ -213,8 +222,19 @@ github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
Expand All @@ -235,6 +255,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ=
Expand All @@ -244,6 +266,8 @@ github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWR
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
Expand Down
13 changes: 11 additions & 2 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
)

type Config struct {
Repository []Repository `yaml:"repository"`
Storage []Storage `yaml:"storage"`
Repository []Repository `yaml:"repository"`
Storage []MultiStorage `yaml:"storage"`
}

type Repository struct {
Expand All @@ -23,6 +23,15 @@ type Storage struct {
Path string `yaml:"path"`
}

type MultiStorage struct {
Storage `mapstructure:",squash"`
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
Region string `yaml:"region"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
}

var Path string

var vp *viper.Viper
Expand Down
18 changes: 13 additions & 5 deletions internal/rip/rip.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"time"
)

func Rip(repo config.Repository, storages []config.Storage) error {
func Rip(repo config.Repository, storages []config.MultiStorage) error {
id := uuid.New().String()
// get current directory
currentDir, _ := os.Getwd()
Expand Down Expand Up @@ -68,16 +68,24 @@ func Rip(repo config.Repository, storages []config.Storage) error {

// handle storages
for _, s := range storages {
var err error
switch s.Type {
case "file":
case storage.FileStorage:
fileBackend := storage.File{}
err := fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes())
err = fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes())
case storage.S3Storage:
s3Backend, err := storage.New(s.Endpoint, s.Bucket, s.Region, s.AccessKeyID, s.SecretAccessKey)
if err != nil {
ui.Errorf("Error storing file, %s", err)
ui.Errorf("Error creating S3 backend, %s", err)
return err
}
ui.Printf("File %s stored", path.Join(s.Path, base))
err = s3Backend.PutObject(base, archive.Bytes())
}
if err != nil {
ui.Errorf("Error storing file, %s", err)
return err
}
ui.Printf("File %s stored", path.Join(s.Path, base))
}

// cleanup
Expand Down
66 changes: 66 additions & 0 deletions internal/storage/s3.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package storage

import (
"bytes"
"context"
"github.com/leslieleung/reaper/internal/ui"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)

var _ Storage = (*S3)(nil)

type S3 struct {
Endpoint string
Bucket string
Region string
AccessKeyID string
SecretAccessKey string

client *minio.Client
}

func New(endpoint, bucket, region, accessKeyID, secretAccessKey string) (*S3, error) {
s3 := &S3{
Endpoint: endpoint,
Bucket: bucket,
Region: region,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
}

client, err := minio.New(s3.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(s3.AccessKeyID, s3.SecretAccessKey, ""),
Secure: true,
})
if err != nil {
return nil, err
}
s3.client = client
return s3, nil
}

func (s S3) ListObject(prefix string) ([]Object, error) {
// TODO implement me
panic("implement me")
}

func (s S3) GetObject(identifier string) (Object, error) {
// TODO implement me
panic("implement me")
}

func (s S3) PutObject(identifier string, data []byte) error {
size := int64(len(data))
info, err := s.client.PutObject(context.Background(), s.Bucket, identifier, bytes.NewReader(data), size, minio.PutObjectOptions{})
if err != nil {
return err
}
ui.Printf("info: %v", info)
return nil
}

func (s S3) DeleteObject(identifier string) error {
// TODO implement me
panic("implement me")
}
5 changes: 5 additions & 0 deletions internal/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ package storage

import "time"

const (
FileStorage = "file"
S3Storage = "s3"
)

type Object struct {
Path string
Content []byte
Expand Down

0 comments on commit b235e69

Please sign in to comment.