From 7ce90da32d3f50c94a433d67f436a6e8276cc912 Mon Sep 17 00:00:00 2001 From: Leslie Leung Date: Sun, 17 Dec 2023 22:02:52 +0800 Subject: [PATCH] feat: add `bury` command, archives all release assets as in #12 --- README.md | 9 +++++ README_zh.md | 9 +++++ cmd/bury/bury.go | 61 ++++++++++++++++++++++++++++++++ cmd/root.go | 2 ++ internal/release/release.go | 63 ++++++++++++++++++++++++++++++++++ internal/rip/repository.go | 5 ++- internal/rip/rip.go | 17 +++------ internal/scm/github/github.go | 34 ++++++++++++++++++ internal/scm/repo.go | 28 +++++++++++++++ internal/storage/storage.go | 22 +++++++++++- internal/typedef/repository.go | 8 +++++ 11 files changed, 242 insertions(+), 16 deletions(-) create mode 100644 cmd/bury/bury.go create mode 100644 internal/release/release.go create mode 100644 internal/scm/repo.go diff --git a/README.md b/README.md index e692e23..cfa83fd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ REpository ArchivER(REAPER) is a tool to archive repositories from any Git serve - [Usage](#usage) - [rip](#rip) - [run](#run) + - [bury](#bury) - [daemon](#daemon) - [Configuration](#configuration) - [Storage](#storage) @@ -85,6 +86,14 @@ reaper run Combined with cron, you can archive repositories periodically. +### bury + +`bury` archives all release assets of a repository. + +```bash +reaper bury reaper +``` + ### daemon `daemon` runs REAPER as a daemon. It will archive all repositories defined in configuration periodically. diff --git a/README_zh.md b/README_zh.md index 040ec9d..af27a1b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -10,6 +10,7 @@ REpository ArchivER(REAPER)是一个用于从任何Git服务器归档 Git - [rip](#rip) - [run](#run) - [daemon](#daemon) + - [bury](#bury) - [配置](#配置) - [存储](#存储) - [使用 Docker 运行](#使用-docker-运行) @@ -85,6 +86,14 @@ reaper run 结合cron,你可以定期归档 Git 仓库。 +### bury + +`bury`命令会归档指定 Git 仓库的所有发布产物。 + +```bash +reaper bury reaper +``` + ### daemon `daemon`命令会启动一个守护进程,它会在后台运行,归档在配置中定义的所有 Git 仓库。 diff --git a/cmd/bury/bury.go b/cmd/bury/bury.go new file mode 100644 index 0000000..10e3279 --- /dev/null +++ b/cmd/bury/bury.go @@ -0,0 +1,61 @@ +package bury + +import ( + "github.com/leslieleung/reaper/internal/config" + "github.com/leslieleung/reaper/internal/release" + "github.com/leslieleung/reaper/internal/rip" + "github.com/leslieleung/reaper/internal/typedef" + "github.com/leslieleung/reaper/internal/ui" + "github.com/spf13/cobra" +) + +var Cmd = &cobra.Command{ + Use: "bury", + Short: "bury immediately downloads all release assets of a repo", + Run: runBury, + Args: cobra.ExactArgs(1), +} + +var storageName string + +func runBury(cmd *cobra.Command, args []string) { + repoName := args[0] + + storageMap := config.GetStorageMap() + storages := make([]typedef.MultiStorage, 0) + if storageName != "" { + if s, ok := storageMap[storageName]; !ok { + ui.Errorf("Storage %s not found in config", storageName) + return + } else { + storages = append(storages, s) + } + } else { + for _, storage := range storageMap { + storages = append(storages, storage) + } + } + + for _, repo := range rip.GetRepositories(repoName) { + storages := make([]typedef.MultiStorage, 0) + for _, storage := range repo.Storage { + if s, ok := storageMap[storage]; !ok { + ui.Errorf("Storage %s not found in config", storage) + continue + } else { + storages = append(storages, s) + } + } + ui.Printf("Running %s", repo.Name) + if err := release.DownloadAllAssets(repo, storages); err != nil { + ui.Errorf("Error running %s, %s", repo.Name, err) + // move on to next repo + } + } + ui.Printf("Done") +} + +func init() { + Cmd.Flags().StringVarP(&storageName, "storage", "s", "", + "storage to use, if not specified, all storages will be used") +} diff --git a/cmd/root.go b/cmd/root.go index c420d32..6409c62 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/leslieleung/reaper/cmd/bury" "github.com/leslieleung/reaper/cmd/daemon" "github.com/leslieleung/reaper/cmd/rip" "github.com/leslieleung/reaper/cmd/run" @@ -27,6 +28,7 @@ func init() { rootCmd.AddCommand(rip.Cmd) rootCmd.AddCommand(run.Cmd) rootCmd.AddCommand(daemon.Cmd) + rootCmd.AddCommand(bury.Cmd) // flags rootCmd.PersistentFlags().StringVarP(&config.Path, "config", "c", "config.yaml", "config file path") } diff --git a/internal/release/release.go b/internal/release/release.go new file mode 100644 index 0000000..d66b6af --- /dev/null +++ b/internal/release/release.go @@ -0,0 +1,63 @@ +package release + +import ( + "fmt" + "github.com/leslieleung/reaper/internal/scm" + "github.com/leslieleung/reaper/internal/scm/github" + "github.com/leslieleung/reaper/internal/storage" + "github.com/leslieleung/reaper/internal/typedef" + "github.com/leslieleung/reaper/internal/ui" + "io" +) + +func DownloadAllAssets(repo typedef.Repository, storages []typedef.MultiStorage) error { + r, err := scm.NewRepository(repo.URL) + if err != nil { + return err + } + c, err := github.New() + if err != nil { + return err + } + // get all releases + releases, err := c.GetReleases(r.Owner, r.Name) + if err != nil { + return err + } + for _, release := range releases { + ui.Printf("Downloading %s", release.GetTagName()) + // get all assets + assets, err := c.GetReleaseAssets(r.Owner, r.Name, release.GetID()) + if err != nil { + return err + } + for _, asset := range assets { + if asset.GetState() != "uploaded" { + continue + } + ui.Printf("Downloading asset %s", asset.GetName()) + path := fmt.Sprintf("%s-%s/%s", repo.Name, release.GetTagName(), asset.GetName()) + // download asset + rc, err := c.DownloadAsset(r.Owner, r.Name, asset.GetID()) + if err != nil { + return err + } + // put rc to file + data, err := io.ReadAll(rc) + if err != nil { + return err + } + for _, s := range storages { + backend, err := storage.GetStorage(s) + if err != nil { + return err + } + err = backend.PutObject(path, data) + if err != nil { + return err + } + } + } + } + return nil +} diff --git a/internal/rip/repository.go b/internal/rip/repository.go index 4b16994..0c28daf 100644 --- a/internal/rip/repository.go +++ b/internal/rip/repository.go @@ -26,7 +26,7 @@ func GetRepositories(name string) []typedef.Repository { } func addRepo(repo typedef.Repository, ret []typedef.Repository) []typedef.Repository { - switch repo.Type { + switch repo.GetType() { case typedef.TypeRepo: ret = append(ret, repo) case typedef.TypeUser, typedef.TypeOrg: @@ -52,8 +52,7 @@ func addRepo(repo typedef.Repository, ret []typedef.Repository) []typedef.Reposi }) } default: - // backward compatibility, default to repo - ret = append(ret, repo) + ui.Errorf("Invalid repository type %s", repo.Type) } return ret } diff --git a/internal/rip/rip.go b/internal/rip/rip.go index e8d05b0..ff7a871 100644 --- a/internal/rip/rip.go +++ b/internal/rip/rip.go @@ -113,19 +113,12 @@ func Rip(repo typedef.Repository, storages []typedef.MultiStorage) error { // handle storages for _, s := range storages { - var err error - switch s.Type { - case storage.FileStorage: - fileBackend := storage.File{} - err = fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes()) - case storage.S3Storage: - s3Backend, err := storage.New(s.Endpoint, s.Bucket, s.Region, s.AccessKeyID, s.SecretAccessKey) - if err != nil { - ui.Errorf("Error creating S3 backend, %s", err) - return err - } - err = s3Backend.PutObject(base, archive.Bytes()) + backend, err := storage.GetStorage(s) + if err != nil { + ui.Errorf("Error getting backend, %s", err) + return err } + err = backend.PutObject(path.Join(s.Path, base), archive.Bytes()) if err != nil { ui.Errorf("Error storing file, %s", err) return err diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index b219c6a..a8a2b68 100644 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -5,6 +5,8 @@ import ( "github.com/google/go-github/v56/github" "github.com/leslieleung/reaper/internal/config" "github.com/leslieleung/reaper/internal/typedef" + "io" + "net/http" "net/url" "sync" ) @@ -53,3 +55,35 @@ func (c *Client) GetRepos(name string, accountType string) ([]string, error) { } return repos, nil } + +func (c *Client) GetReleases(owner, repo string) ([]*github.RepositoryRelease, error) { + var ( + list []*github.RepositoryRelease + err error + ) + list, _, err = c.c.Repositories.ListReleases(context.Background(), owner, repo, nil) + if err != nil { + return nil, err + } + return list, nil +} + +func (c *Client) GetReleaseAssets(owner, repo string, id int64) ([]*github.ReleaseAsset, error) { + var ( + list []*github.ReleaseAsset + err error + ) + list, _, err = c.c.Repositories.ListReleaseAssets(context.Background(), owner, repo, id, nil) + if err != nil { + return nil, err + } + return list, nil +} + +func (c *Client) DownloadAsset(owner, repo string, id int64) (io.ReadCloser, error) { + rc, _, err := c.c.Repositories.DownloadReleaseAsset(context.Background(), owner, repo, id, http.DefaultClient) + if err != nil { + return nil, err + } + return rc, nil +} diff --git a/internal/scm/repo.go b/internal/scm/repo.go new file mode 100644 index 0000000..f47dc2b --- /dev/null +++ b/internal/scm/repo.go @@ -0,0 +1,28 @@ +package scm + +import ( + "errors" + "strings" +) + +type Repository struct { + Host string + Owner string + Name string +} + +var ( + ErrInvalidURL = errors.New("invalid url") +) + +func NewRepository(url string) (*Repository, error) { + r := &Repository{} + l := strings.Split(url, "/") + if len(l) < 3 { + return nil, ErrInvalidURL + } + r.Host = l[0] + r.Owner = l[1] + r.Name = l[2] + return r, nil +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ae91d5a..d7e52dd 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,6 +1,10 @@ package storage -import "time" +import ( + "errors" + "github.com/leslieleung/reaper/internal/typedef" + "time" +) const ( FileStorage = "file" @@ -23,3 +27,19 @@ type Storage interface { // DeleteObject deletes the object identified by the given identifier. DeleteObject(identifier string) error } + +func GetStorage(storage typedef.MultiStorage) (Storage, error) { + var ( + backend Storage + err error + ) + switch storage.Type { + case FileStorage: + backend = &File{} + case S3Storage: + backend, err = New(storage.Endpoint, storage.Bucket, storage.Region, storage.AccessKeyID, storage.SecretAccessKey) + default: + err = errors.New("unknown storage type") + } + return backend, err +} diff --git a/internal/typedef/repository.go b/internal/typedef/repository.go index 328939a..178ba34 100644 --- a/internal/typedef/repository.go +++ b/internal/typedef/repository.go @@ -9,3 +9,11 @@ type Repository struct { Type string `yaml:"type"` // repo, user, org (default: repo) OrgName string `yaml:"orgName"` } + +func (r *Repository) GetType() string { + // backward compatibility, default to repo + if r.Type == "" { + return TypeRepo + } + return r.Type +}