Merge pull request #44 from replicatedhq/divolgin/airgap-upload

airgap upload

Makefile

@@ -16,11 +16,11 @@ ffi: fmt vet
 
 .PHONY: fmt
 fmt:
-	go fmt ./pkg/... ./cmd/...
+	go fmt ./pkg/... ./cmd/... ./ffi/...
 
 .PHONY: vet
 vet:
-	go vet ./pkg/... ./cmd/...
+	go vet ./pkg/... ./cmd/... ./ffi/...
 
 .PHONY: gosec
 gosec:

ffi/airgap.go

@@ -5,14 +5,16 @@ import "C"
 import (
 	"archive/tar"
 	"compress/gzip"
-	"os/exec"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 
+	"github.com/docker/distribution/reference"
 	"github.com/mholt/archiver"
 	"github.com/pkg/errors"
 	kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1"
@@ -21,21 +23,90 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 )
 
-//export PullFromAirgap
-func PullFromAirgap(licenseData string, airgapURL string, downstream string, outputFile string) int {
-	workspace, err := ioutil.TempDir("", "kots-airgap")
+type ImageRef struct {
+	Domain string
+	Name   string
+	Tag    string
+	Digest string
+}
+
+//export RewriteAndPushImageName
+func RewriteAndPushImageName(imageFile, image, registryHost, registryOrg, username, password string) int {
+	imageRef, err := parseImageRef(image)
 	if err != nil {
-		fmt.Printf("failed to create temp dir: %s\n", err)
+		fmt.Printf("failed to parse image %s: %s\n", image, err)
+		return 1
+	}
+	localImage := imageRefToString(imageRef, registryHost, registryOrg)
+
+	cmdArgs := []string{
+		"copy",
+		"--dest-tls-verify=false",
+	}
+	if len(username) > 0 && len(password) > 0 {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--dest-creds=%s:%s", username, password))
+	}
+	cmdArgs = append(cmdArgs,
+		fmt.Sprintf("oci-archive:%s", imageFile),
+		fmt.Sprintf("docker://%s", localImage),
+	)
+
+	cmd := exec.Command("skopeo", cmdArgs...)
+	cmdOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf("run failed with output: %s\n", cmdOutput)
 		return 1
 	}
-	// defer os.RemoveAll(workspace)
 
-	// airgapDir contains release tar and all images as individual tars
-	airgapDir, err := downloadAirgapAchive(workspace, airgapURL)
+	return 0
+}
+
+func parseImageRef(image string) (*ImageRef, error) {
+	ref := &ImageRef{}
+
+	// named, err := reference.ParseNormalizedNamed(image)
+	parsed, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse image name %q", image)
+	}
+
+	if named, ok := parsed.(reference.Named); ok {
+		ref.Domain = reference.Domain(named)
+		ref.Name = named.Name()
+	} else {
+		return nil, errors.New(fmt.Sprintf("unsupported ref type: %T", parsed))
+	}
+
+	if tagged, ok := parsed.(reference.Tagged); ok {
+		ref.Tag = tagged.Tag()
+	} else if can, ok := parsed.(reference.Canonical); ok {
+		ref.Digest = can.Digest().String()
+	} else {
+		ref.Tag = "latest"
+	}
+
+	return ref, nil
+}
+
+func imageRefToString(ref *ImageRef, registryHost, registryOrg string) string {
+	pathParts := strings.Split(ref.Name, "/")
+	imageName := fmt.Sprintf("%s/%s", registryOrg, pathParts[len(pathParts)-1])
+
+	// there might be a way to do this with reference package too
+	if ref.Digest != "" {
+		return fmt.Sprintf("%s/%s@sha256:%s", registryHost, imageName, ref.Digest)
+	}
+	return fmt.Sprintf("%s/%s:%s", registryHost, imageName, ref.Tag)
+}
+
+//export PullFromAirgap
+func PullFromAirgap(licenseData, airgapDir, downstream, outputFile, registryHost, registryNamesapce string) int {
+	workspace, err := ioutil.TempDir("", "kots-airgap")
 	if err != nil {
-		fmt.Printf("failed to download airgap archive: %s\n", err)
+		fmt.Printf("failed to create temp dir: %s\n", err)
 		return 1
 	}
+	defer os.RemoveAll(workspace)
 
 	// releaseDir is the contents of the release tar (yaml, no images)
 	releaseDir, err := extractAppRelease(workspace, airgapDir)
@@ -44,8 +115,6 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out
 		return 1
 	}
 
-	///............
-
 	kotsscheme.AddToScheme(scheme.Scheme)
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	obj, _, err := decode([]byte(licenseData), nil, nil)
@@ -73,7 +142,7 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out
 		fmt.Printf("failed to create temp root path: %s\n", err.Error())
 		return 1
 	}
-	// defer os.RemoveAll(tmpRoot)
+	defer os.RemoveAll(tmpRoot)
 
 	pullOptions := pull.PullOptions{
 		Downstreams:         []string{downstream},
@@ -82,6 +151,11 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out
 		ExcludeKotsKinds:    true,
 		RootDir:             tmpRoot,
 		ExcludeAdminConsole: true,
+		RewriteImages: pull.RewriteImages{
+			ImageFiles: filepath.Join(airgapDir, "images"),
+			Host:       registryHost,
+			Namespace:  registryNamesapce,
+		},
 	}
 
 	if _, err := pull.Pull(fmt.Sprintf("replicated://%s", license.Spec.AppSlug), pullOptions); err != nil {
@@ -109,11 +183,6 @@ func PullFromAirgap(licenseData string, airgapURL string, downstream string, out
 		return 1
 	}
 
-	if err := pushImages(filepath.Join(airgapDir, "images"), []string{}); err != nil {
-		fmt.Printf("unable to push images: %s\n", err.Error())
-		return 1
-	}
-
 	return 0
 }
 
@@ -256,68 +325,4 @@ func extractOneArchive(tgzFile string, destDir string) error {
 	}
 
 	return nil
-}
-
-func pushImages(srcDir string, imageNameParts []string) error {
-	files, err := ioutil.ReadDir(srcDir)
-	if err != nil {
-		return errors.Wrapf(err, "failed to list image files")
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			// this function will modify the array argument
-			err := pushImages(filepath.Join(srcDir, file.Name()), append(imageNameParts, file.Name()))
-			if err != nil {
-				return errors.Wrapf(err, "failed to push images")
-			}
-		} else {
-			// this function will modify the array argument
-			if err := pushImageFromFile(filepath.Join(srcDir, file.Name()), append(imageNameParts, file.Name())); err != nil {
-				return errors.Wrapf(err, "failed to push image")
-			}
-		}
-	}
-
-	return nil
-}
-
-func pushImageFromFile(filename string, imageNameParts []string) error {
-	// TODO: don't hardcode registry name
-	imageName, err := imageNameFromNameParts("image-registry-lb:5000", imageNameParts)
-	if err != nil {
-		return errors.Wrapf(err, "failed to generate image name from %v", imageNameParts)
-	}
-	cmd := exec.Command("skopeo", "copy", "--dest-tls-verify=false", fmt.Sprintf("oci-archive:%s", filename), fmt.Sprintf("docker://%s", imageName))
-	stdoutStderr, err := cmd.CombinedOutput()
-	if err != nil {
-		fmt.Printf("run failed with output: %s\n", stdoutStderr)
-		return errors.Wrap(err, "failed to execute skopeo")
-	}
-
-	return nil
-}
-
-func imageNameFromNameParts(registry string, nameParts []string) (string, error) {
-	// imageNameParts looks like this:
-	// ["quay.io", "someorg", "imagename", "imagetag"]
-	// or
-	// ["quay.io", "someorg", "imagename", "sha256", "<sha>"]
-	// we want to replace host with local registry and build image name from the remaining parts
-
-	if len(nameParts) < 4 {
-		return "", fmt.Errorf("not enough parts in image name: %v", nameParts)
-	}
-
-	var name, tag string
-	nameParts[0] = registry
-	if nameParts[len(nameParts) - 2] == "sha256" {
-		tag = fmt.Sprintf("@sha256:%s", nameParts[len(nameParts) - 1])
-		name = filepath.Join(nameParts[:len(nameParts)-2]...)
-	} else {
-		tag = fmt.Sprintf(":%s", nameParts[len(nameParts) - 1])
-		name = filepath.Join(nameParts[:len(nameParts)-1]...)
-	}
-
-	return fmt.Sprintf("%s%s", name, tag), nil
 }

ffi/online.go

@@ -80,4 +80,3 @@ func PullFromLicense(licenseData string, downstream string, outputFile string) i
 
 	return 0
 }
-

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd // indirect
 	github.com/cloudflare/cfssl v0.0.0-20190808011637-b1ec8c586c2a // indirect
 	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
+	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect

go.sum

@@ -131,6 +131,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dimchansky/utfbom v1.0.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v0.0.0-20180920040454-5637cf3d8a31/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
@@ -505,6 +506,7 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=