feat: add option to create remote repository before first push (#43)

* feat: add option to create remote repository before first push

fix lint

* repository: change interface to only expose Ensure method

* fix go.mod

go.mod

@@ -1,6 +1,6 @@
 module github.com/tsuru/deploy-agent
 
-go 1.19
+go 1.21
 
 require (
 	github.com/alessio/shellescape v1.4.1
@@ -9,7 +9,8 @@ require (
 	github.com/docker/docker v23.0.0-rc.1+incompatible
 	github.com/google/go-containerregistry v0.12.0
 	github.com/moby/buildkit v0.11.3
-	github.com/stretchr/testify v1.8.0
+	github.com/oracle/oci-go-sdk/v65 v65.73.0
+	github.com/stretchr/testify v1.8.4
 	golang.org/x/sync v0.7.0
 	google.golang.org/grpc v1.65.0
 	google.golang.org/protobuf v1.34.1
@@ -61,6 +62,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa // indirect
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect

main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/tsuru/deploy-agent/pkg/build/buildkit"
 	buildpb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
 	"github.com/tsuru/deploy-agent/pkg/health"
+	"github.com/tsuru/deploy-agent/pkg/repository"
 )
 
 const (
@@ -42,12 +43,13 @@ var cfg struct {
 	BuildKitAutoDiscoveryKubernetesLeasePrefix                string
 	BuildKitAutoDiscoveryStatefulset                          string
 	KubernetesConfig                                          string
-	BuildKitAutoDiscoveryScaleGracefulPeriod                  time.Duration
+	RemoteRepositoryPath                                      string
 	BuildKitAutoDiscoveryTimeout                              time.Duration
 	BuildKitAutoDiscoveryKubernetesPort                       int
 	Port                                                      int
 	ServerMaxRecvMsgSize                                      int
 	ServerMaxSendMsgSize                                      int
+	BuildKitAutoDiscoveryScaleGracefulPeriod                  time.Duration
 	BuildKitAutoDiscovery                                     bool
 	BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels          bool
 	BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp bool
@@ -65,6 +67,8 @@ func main() {
 	flag.StringVar(&cfg.BuildkitAddress, "buildkit-addr", getEnvOrDefault("BUILDKIT_HOST", ""), "Buildkit server address")
 	flag.StringVar(&cfg.BuildkitTmpDir, "buildkit-tmp-dir", os.TempDir(), "Directory path to store temp files during container image builds")
 
+	flag.StringVar(&cfg.RemoteRepositoryPath, "remote-repository-path", getEnvOrDefault("REMOTE_REPOSITORY_PATH", ""), "Remote image repository providers config path")
+
 	flag.BoolVar(&cfg.BuildKitAutoDiscovery, "buildkit-autodiscovery", false, "Whether should dynamically discover the BuildKit service based on Tsuru app (if any)")
 	flag.DurationVar(&cfg.BuildKitAutoDiscoveryTimeout, "buildkit-autodiscovery-timeout", (5 * time.Minute), "Max duration to discover an available BuildKit")
 	flag.StringVar(&cfg.BuildKitAutoDiscoveryKubernetesPodSelector, "buildkit-autodiscovery-kubernetes-pod-selector", "", "Label selector of BuildKit's pods on Kubernetes")
@@ -148,6 +152,17 @@ func newBuildKit() (*buildkit.BuildKit, error) {
 		c = bc
 	}
 
+	if cfg.RemoteRepositoryPath != "" {
+		repositoryData, err := os.ReadFile(cfg.RemoteRepositoryPath)
+		if err != nil {
+			return nil, err
+		}
+		opts.RemoteRepository, err = repository.NewRemoteRepository(repositoryData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to handle remote repository cfg: %w", err)
+		}
+	}
+
 	b := buildkit.NewBuildKit(c, opts)
 
 	if cfg.BuildKitAutoDiscovery {

pkg/build/buildkit/build.go

@@ -40,12 +40,14 @@ import (
 	"github.com/tsuru/deploy-agent/pkg/build"
 	"github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler"
 	pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
+	repo "github.com/tsuru/deploy-agent/pkg/repository"
 	"github.com/tsuru/deploy-agent/pkg/util"
 )
 
 var _ build.Builder = (*BuildKit)(nil)
 
 type BuildKitOptions struct {
+	RemoteRepository             map[string]repo.Repository
 	TempDir                      string
 	DiscoverBuildKitClientForApp bool
 }
@@ -166,6 +168,13 @@ func (b *BuildKit) buildFromAppSourceFiles(ctx context.Context, c *client.Client
 	}
 	defer cleanFunc()
 
+	if b.opts.RemoteRepository != nil {
+		err = b.createRemoteRepository(ctx, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
 		return nil, err
 	}
@@ -222,6 +231,13 @@ func (b *BuildKit) buildFromContainerImage(ctx context.Context, c *client.Client
 	}
 	defer cleanFunc()
 
+	if b.opts.RemoteRepository != nil {
+		err = b.createRemoteRepository(ctx, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
 		return nil, err
 	}
@@ -255,6 +271,18 @@ func (b *BuildKit) extractTsuruConfigsFromContainerImage(ctx context.Context, c
 	return callBuildKitToExtractTsuruConfigs(ctx, c, tmpDir, workingDir)
 }
 
+func (b *BuildKit) createRemoteRepository(ctx context.Context, r *pb.BuildRequest) error {
+	for _, v := range r.DestinationImages {
+		if provider, ok := b.opts.RemoteRepository[build.GetRegistry(v)]; ok {
+			err := provider.Ensure(ctx, v)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func extractContainerImageConfigFromImageManifest(ctx context.Context, imageStr string, insecureRegistry bool) (*pb.ContainerImageConfig, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -398,6 +426,13 @@ func (b *BuildKit) buildFromContainerFile(ctx context.Context, c *client.Client,
 	}
 	defer cleanFunc()
 
+	if b.opts.RemoteRepository != nil {
+		err = b.createRemoteRepository(ctx, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
 		return nil, err
 	}
@@ -428,7 +463,12 @@ func (b *BuildKit) buildPlatform(ctx context.Context, c *client.Client, r *pb.Bu
 		return err
 	}
 	defer cleanFunc()
-
+	if b.opts.RemoteRepository != nil {
+		err = b.createRemoteRepository(ctx, r)
+		if err != nil {
+			return err
+		}
+	}
 	return callBuildKitBuild(ctx, c, tmpDir, r, w)
 }
 

pkg/build/buildkit/build_test.go

@@ -26,6 +26,8 @@ import (
 
 	. "github.com/tsuru/deploy-agent/pkg/build/buildkit"
 	pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
+	"github.com/tsuru/deploy-agent/pkg/repository"
+	"github.com/tsuru/deploy-agent/pkg/repository/fake"
 	"github.com/tsuru/deploy-agent/pkg/util"
 )
 
@@ -376,6 +378,28 @@ func TestBuildKit_Build_FromContainerImages(t *testing.T) {
 		}, appFiles)
 	})
 
+	t.Run("creating remote repository", func(t *testing.T) {
+		req := &pb.BuildRequest{
+			Kind: pb.BuildKind_BUILD_KIND_APP_DEPLOY_WITH_CONTAINER_IMAGE,
+			App: &pb.TsuruApp{
+				Name: "my-app",
+			},
+			SourceImage:       "nginx:1.22-alpine",
+			DestinationImages: []string{baseRegistry(t, "app-my-app", "v1")},
+			PushOptions:       &pb.PushOptions{InsecureRegistry: registryHTTP},
+		}
+		opts := &BuildKitOptions{TempDir: t.TempDir(), RemoteRepository: map[string]repository.Repository{registryAddress: &fake.FakeRepository{}}}
+		assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool(nil))
+		_, err := NewBuildKit(bc, *opts).
+			Build(context.TODO(), req, os.Stdout)
+		require.NoError(t, err)
+		assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool{baseRegistry(t, "app-my-app", "v1"): true})
+		_, err = NewBuildKit(bc, *opts).
+			Build(context.TODO(), req, os.Stdout)
+		require.NoError(t, err)
+		assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool{baseRegistry(t, "app-my-app", "v1"): true})
+	})
+
 	t.Run("container image without Tsuru app files (tsuru.yaml, Procfile) + job image push", func(t *testing.T) {
 		req := &pb.BuildRequest{
 			Kind: pb.BuildKind_BUILD_KIND_JOB_CREATE_WITH_CONTAINER_IMAGE,

pkg/build/helpers.go

@@ -293,3 +293,17 @@ func SortExposedPorts(ports map[string]struct{}) []string {
 
 	return ps
 }
+
+func GetRegistry(image string) string {
+	defaultRegistry := "docker.io"
+	if !strings.Contains(image, "/") {
+		return defaultRegistry
+	}
+
+	registry := strings.Split(image, "/")[0]
+	if strings.Contains(registry, ".") || strings.Contains(registry, ":") {
+		return registry
+	}
+
+	return defaultRegistry
+}