feat: add bury command, archives all release assets as in #12

README.md

@@ -9,6 +9,7 @@ REpository ArchivER(REAPER) is a tool to archive repositories from any Git serve
 - [Usage](#usage)
   - [rip](#rip)
   - [run](#run)
+  - [bury](#bury)
   - [daemon](#daemon)
 - [Configuration](#configuration)
 - [Storage](#storage)
@@ -85,6 +86,14 @@ reaper run
 
 Combined with cron, you can archive repositories periodically.
 
+### bury
+
+`bury` archives all release assets of a repository.
+
+```bash
+reaper bury reaper
+```
+
 ### daemon
 
 `daemon` runs REAPER as a daemon. It will archive all repositories defined in configuration periodically.

README_zh.md

@@ -10,6 +10,7 @@ REpository ArchivER（REAPER）是一个用于从任何Git服务器归档 Git 
   - [rip](#rip)
   - [run](#run)
   - [daemon](#daemon)
+  - [bury](#bury)
 - [配置](#配置)
 - [存储](#存储)
 - [使用 Docker 运行](#使用-docker-运行)
@@ -85,6 +86,14 @@ reaper run
 
 结合cron，你可以定期归档 Git 仓库。
 
+### bury
+
+`bury`命令会归档指定 Git 仓库的所有发布产物。
+
+```bash
+reaper bury reaper
+```
+
 ### daemon
 
 `daemon`命令会启动一个守护进程，它会在后台运行，归档在配置中定义的所有 Git 仓库。

cmd/bury/bury.go

@@ -0,0 +1,61 @@
+package bury
+
+import (
+	"github.com/leslieleung/reaper/internal/config"
+	"github.com/leslieleung/reaper/internal/release"
+	"github.com/leslieleung/reaper/internal/rip"
+	"github.com/leslieleung/reaper/internal/typedef"
+	"github.com/leslieleung/reaper/internal/ui"
+	"github.com/spf13/cobra"
+)
+
+var Cmd = &cobra.Command{
+	Use:   "bury",
+	Short: "bury immediately downloads all release assets of a repo",
+	Run:   runBury,
+	Args:  cobra.ExactArgs(1),
+}
+
+var storageName string
+
+func runBury(cmd *cobra.Command, args []string) {
+	repoName := args[0]
+
+	storageMap := config.GetStorageMap()
+	storages := make([]typedef.MultiStorage, 0)
+	if storageName != "" {
+		if s, ok := storageMap[storageName]; !ok {
+			ui.Errorf("Storage %s not found in config", storageName)
+			return
+		} else {
+			storages = append(storages, s)
+		}
+	} else {
+		for _, storage := range storageMap {
+			storages = append(storages, storage)
+		}
+	}
+
+	for _, repo := range rip.GetRepositories(repoName) {
+		storages := make([]typedef.MultiStorage, 0)
+		for _, storage := range repo.Storage {
+			if s, ok := storageMap[storage]; !ok {
+				ui.Errorf("Storage %s not found in config", storage)
+				continue
+			} else {
+				storages = append(storages, s)
+			}
+		}
+		ui.Printf("Running %s", repo.Name)
+		if err := release.DownloadAllAssets(repo, storages); err != nil {
+			ui.Errorf("Error running %s, %s", repo.Name, err)
+			// move on to next repo
+		}
+	}
+	ui.Printf("Done")
+}
+
+func init() {
+	Cmd.Flags().StringVarP(&storageName, "storage", "s", "",
+		"storage to use, if not specified, all storages will be used")
+}

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/leslieleung/reaper/cmd/bury"
 	"github.com/leslieleung/reaper/cmd/daemon"
 	"github.com/leslieleung/reaper/cmd/rip"
 	"github.com/leslieleung/reaper/cmd/run"
@@ -27,6 +28,7 @@ func init() {
 	rootCmd.AddCommand(rip.Cmd)
 	rootCmd.AddCommand(run.Cmd)
 	rootCmd.AddCommand(daemon.Cmd)
+	rootCmd.AddCommand(bury.Cmd)
 	// flags
 	rootCmd.PersistentFlags().StringVarP(&config.Path, "config", "c", "config.yaml", "config file path")
 }

internal/release/release.go

@@ -0,0 +1,63 @@
+package release
+
+import (
+	"fmt"
+	"github.com/leslieleung/reaper/internal/scm"
+	"github.com/leslieleung/reaper/internal/scm/github"
+	"github.com/leslieleung/reaper/internal/storage"
+	"github.com/leslieleung/reaper/internal/typedef"
+	"github.com/leslieleung/reaper/internal/ui"
+	"io"
+)
+
+func DownloadAllAssets(repo typedef.Repository, storages []typedef.MultiStorage) error {
+	r, err := scm.NewRepository(repo.URL)
+	if err != nil {
+		return err
+	}
+	c, err := github.New()
+	if err != nil {
+		return err
+	}
+	// get all releases
+	releases, err := c.GetReleases(r.Owner, r.Name)
+	if err != nil {
+		return err
+	}
+	for _, release := range releases {
+		ui.Printf("Downloading %s", release.GetTagName())
+		// get all assets
+		assets, err := c.GetReleaseAssets(r.Owner, r.Name, release.GetID())
+		if err != nil {
+			return err
+		}
+		for _, asset := range assets {
+			if asset.GetState() != "uploaded" {
+				continue
+			}
+			ui.Printf("Downloading asset %s", asset.GetName())
+			path := fmt.Sprintf("%s-%s/%s", repo.Name, release.GetTagName(), asset.GetName())
+			// download asset
+			rc, err := c.DownloadAsset(r.Owner, r.Name, asset.GetID())
+			if err != nil {
+				return err
+			}
+			// put rc to file
+			data, err := io.ReadAll(rc)
+			if err != nil {
+				return err
+			}
+			for _, s := range storages {
+				backend, err := storage.GetStorage(s)
+				if err != nil {
+					return err
+				}
+				err = backend.PutObject(path, data)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

internal/rip/repository.go

@@ -26,7 +26,7 @@ func GetRepositories(name string) []typedef.Repository {
 }
 
 func addRepo(repo typedef.Repository, ret []typedef.Repository) []typedef.Repository {
-	switch repo.Type {
+	switch repo.GetType() {
 	case typedef.TypeRepo:
 		ret = append(ret, repo)
 	case typedef.TypeUser, typedef.TypeOrg:
@@ -52,8 +52,7 @@ func addRepo(repo typedef.Repository, ret []typedef.Repository) []typedef.Reposi
 			})
 		}
 	default:
-		// backward compatibility, default to repo
-		ret = append(ret, repo)
+		ui.Errorf("Invalid repository type %s", repo.Type)
 	}
 	return ret
 }

internal/rip/rip.go

@@ -113,19 +113,12 @@ func Rip(repo typedef.Repository, storages []typedef.MultiStorage) error {
 
 	// handle storages
 	for _, s := range storages {
-		var err error
-		switch s.Type {
-		case storage.FileStorage:
-			fileBackend := storage.File{}
-			err = fileBackend.PutObject(path.Join(s.Path, base), archive.Bytes())
-		case storage.S3Storage:
-			s3Backend, err := storage.New(s.Endpoint, s.Bucket, s.Region, s.AccessKeyID, s.SecretAccessKey)
-			if err != nil {
-				ui.Errorf("Error creating S3 backend, %s", err)
-				return err
-			}
-			err = s3Backend.PutObject(base, archive.Bytes())
+		backend, err := storage.GetStorage(s)
+		if err != nil {
+			ui.Errorf("Error getting backend, %s", err)
+			return err
 		}
+		err = backend.PutObject(path.Join(s.Path, base), archive.Bytes())
 		if err != nil {
 			ui.Errorf("Error storing file, %s", err)
 			return err

internal/scm/github/github.go

@@ -5,6 +5,8 @@ import (
 	"github.com/google/go-github/v56/github"
 	"github.com/leslieleung/reaper/internal/config"
 	"github.com/leslieleung/reaper/internal/typedef"
+	"io"
+	"net/http"
 	"net/url"
 	"sync"
 )
@@ -53,3 +55,35 @@ func (c *Client) GetRepos(name string, accountType string) ([]string, error) {
 	}
 	return repos, nil
 }
+
+func (c *Client) GetReleases(owner, repo string) ([]*github.RepositoryRelease, error) {
+	var (
+		list []*github.RepositoryRelease
+		err  error
+	)
+	list, _, err = c.c.Repositories.ListReleases(context.Background(), owner, repo, nil)
+	if err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+func (c *Client) GetReleaseAssets(owner, repo string, id int64) ([]*github.ReleaseAsset, error) {
+	var (
+		list []*github.ReleaseAsset
+		err  error
+	)
+	list, _, err = c.c.Repositories.ListReleaseAssets(context.Background(), owner, repo, id, nil)
+	if err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+func (c *Client) DownloadAsset(owner, repo string, id int64) (io.ReadCloser, error) {
+	rc, _, err := c.c.Repositories.DownloadReleaseAsset(context.Background(), owner, repo, id, http.DefaultClient)
+	if err != nil {
+		return nil, err
+	}
+	return rc, nil
+}

internal/scm/repo.go

@@ -0,0 +1,28 @@
+package scm
+
+import (
+	"errors"
+	"strings"
+)
+
+type Repository struct {
+	Host  string
+	Owner string
+	Name  string
+}
+
+var (
+	ErrInvalidURL = errors.New("invalid url")
+)
+
+func NewRepository(url string) (*Repository, error) {
+	r := &Repository{}
+	l := strings.Split(url, "/")
+	if len(l) < 3 {
+		return nil, ErrInvalidURL
+	}
+	r.Host = l[0]
+	r.Owner = l[1]
+	r.Name = l[2]
+	return r, nil
+}

internal/storage/storage.go

@@ -1,6 +1,10 @@
 package storage
 
-import "time"
+import (
+	"errors"
+	"github.com/leslieleung/reaper/internal/typedef"
+	"time"
+)
 
 const (
 	FileStorage = "file"
@@ -23,3 +27,19 @@ type Storage interface {
 	// DeleteObject deletes the object identified by the given identifier.
 	DeleteObject(identifier string) error
 }
+
+func GetStorage(storage typedef.MultiStorage) (Storage, error) {
+	var (
+		backend Storage
+		err     error
+	)
+	switch storage.Type {
+	case FileStorage:
+		backend = &File{}
+	case S3Storage:
+		backend, err = New(storage.Endpoint, storage.Bucket, storage.Region, storage.AccessKeyID, storage.SecretAccessKey)
+	default:
+		err = errors.New("unknown storage type")
+	}
+	return backend, err
+}

internal/typedef/repository.go

@@ -9,3 +9,11 @@ type Repository struct {
 	Type     string   `yaml:"type"` // repo, user, org (default: repo)
 	OrgName  string   `yaml:"orgName"`
 }
+
+func (r *Repository) GetType() string {
+	// backward compatibility, default to repo
+	if r.Type == "" {
+		return TypeRepo
+	}
+	return r.Type
+}