WIP - add first class secrets support

api/api-rules/violation_exceptions.list

@@ -3,6 +3,7 @@ API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/ap
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildSpec,Images
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildSpec,ImportRemoteBuildCache
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildSpec,RegistryAuth
+API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildSpec,Secrets
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildStatus,Conditions
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildStatus,Transitions
 API rule violation: list_type_missing,github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1,ImageBuildStatusTransitionMessage,ImageURLs

deployments/crds/hephaestus.dominodatalab.com_imagebuilds.yaml

@@ -125,6 +125,17 @@ spec:
                       type: string
                   type: object
                 type: array
+              secrets:
+                description: Secrets provides references to Kubernetes secrets to
+                  expose to individual image builds.
+                items:
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                  type: object
+                type: array
             type: object
           status:
             properties:

examples/resources/imagebuild.yaml

@@ -8,6 +8,9 @@ spec:
     - username/repo:tag
   buildArgs:
     - ENV=development
+  secrets:
+    - name: mySecret
+      namespace: default
   cacheMode: min
   cacheTag: local-test
   disableCacheExports: false

pkg/api/hephaestus/v1/imagebuild_types.go

@@ -32,6 +32,8 @@ type ImageBuildSpec struct {
 	DisableLocalBuildCache bool `json:"disableBuildCache,omitempty"`
 	// DisableCacheLayerExport will remove the "inline" cache metadata from the image configuration.
 	DisableCacheLayerExport bool `json:"disableCacheExport,omitempty"`
+	// Secrets provides references to Kubernetes secrets to expose to individual image builds.
+	Secrets []SecretReference `json:"secrets,omitempty"`
 }
 
 type ImageBuildTransition struct {

pkg/api/hephaestus/v1/types.go

@@ -26,6 +26,12 @@ const (
 	PhaseFailed Phase = "Failed"
 )
 
+const (
+	// Kubernetes metadata set by clients required to allow reading secrets by Hephaestus.
+	// Safeguards against accidental secret exposure / exfiltration.
+	AccessLabel = "hephaestus-access"
+)
+
 type BasicAuthCredentials struct {
 	Username string `json:"username,omitempty"`
 	Password string `json:"password,omitempty"`
@@ -49,6 +55,11 @@ type RegistryCredentials struct {
 	Secret        *SecretCredentials    `json:"secret,omitempty"`
 }
 
+type SecretReference struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+}
+
 // ImageBuildStatusTransitionMessage contains information about ImageBuild status transitions.
 //
 // This type is used to publish JSON-formatted messages to one or more configured messaging

pkg/buildkit/buildkit.go

@@ -104,6 +104,7 @@ type BuildOptions struct {
 	ImportCache              []string
 	DisableInlineCacheExport bool
 	Secrets                  map[string]string
+	SecretsData              map[string][]byte
 	FetchAndExtractTimeout   time.Duration
 }
 
@@ -177,6 +178,11 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error {
 		secrets[name] = contents
 	}
 
+	// merge in preloaded data
+	for name, contents := range opts.SecretsData {
+		secrets[name] = contents
+	}
+
 	// build solve options
 	solveOpt := bkclient.SolveOpt{
 		Frontend:      "dockerfile.v0",

pkg/controller/imagebuild/component/builddispatcher.go

@@ -4,14 +4,18 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/dominodatalab/controller-util/core"
 	"github.com/go-logr/logr"
 	"github.com/newrelic/go-agent/v3/newrelic"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	hephv1 "github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1"
 	"github.com/dominodatalab/hephaestus/pkg/buildkit"
@@ -181,6 +185,54 @@ func (c *BuildDispatcherComponent) Reconcile(ctx *core.Context) (ctrl.Result, er
 	}
 	clientInitSeg.End()
 
+	// TODO: refactor this secret extraction to a helper like the credentials persist call
+	// configDir, helpMessage, err := credentials.Persist(ctx, buildLog, ctx.Config, obj.Spec.RegistryAuth)
+	clientset, err := kubernetes.NewForConfig(ctx.Config)
+	if err != nil {
+		return ctrl.Result{}, c.phase.SetFailed(ctx, obj, fmt.Errorf("failure to get kubernetes client: %w", err))
+	}
+
+	// TODO: add logging and NewRelic stuff
+	// TODO: does it make more sense to project the secret into the filesystem somehow and then just use existing secrets?
+	// Extracts secrets into data to pass to buildkit
+	secretsData := make(map[string][]byte)
+	for _, secretRef := range obj.Spec.Secrets {
+		client := clientset.CoreV1().Secrets(secretRef.Namespace)
+
+		secret, err := client.Get(ctx, secretRef.Name, metav1.GetOptions{})
+
+		if err != nil {
+			return ctrl.Result{}, c.phase.SetFailed(ctx, obj,
+				fmt.Errorf("failure to load secret %q/%q: %w", secretRef.Namespace, secretRef.Name, err))
+		}
+
+		// prevent exfiltration of arbitrary secret values by using the presence of this lable
+		if _, ok := secret.Labels[hephv1.AccessLabel]; !ok {
+			return ctrl.Result{}, c.phase.SetFailed(ctx, obj,
+				fmt.Errorf("secret %q/%q requires label %q", secretRef.Namespace, secretRef.Name, hephv1.AccessLabel))
+		}
+
+		// TODO: verify ImageBuild takes ownership of the secret and cleans it up after use to prevent cluster cruft
+		// https://kubernetes.io/blog/2021/05/14/using-finalizers-to-control-deletion/
+		// NOTE:
+		if err = controllerutil.SetOwnerReference(obj, secret, ctx.Scheme); err != nil {
+			// non-fatal error thats logged but ignored
+			log.Error(err, "Ignoring error taking ownership of secret %q/%q", secretRef.Namespace, secretRef.Name)
+		}
+		// TODO: this is an alternative way to manually cook up metadata rather than just using controllerutil
+		// secret.OwnerReferences = append(secret.OwnerReferences)
+		// _, err = client.Update(ctx, secret, metav1.UpdateOptions{})
+		// if err != nil {
+		// 	log.Info("")
+		// }
+
+		// builds a path from the secret name and the key from the data array like {name}/{key}
+		for path, data := range secret.Data {
+			name := strings.Join([]string{secret.Name, path}, "/")
+			secretsData[name] = data
+		}
+	}
+
 	buildOpts := buildkit.BuildOptions{
 		Context:                  obj.Spec.Context,
 		Images:                   obj.Spec.Images,
@@ -189,6 +241,7 @@ func (c *BuildDispatcherComponent) Reconcile(ctx *core.Context) (ctrl.Result, er
 		ImportCache:              obj.Spec.ImportRemoteBuildCache,
 		DisableInlineCacheExport: obj.Spec.DisableCacheLayerExport,
 		Secrets:                  c.cfg.Secrets,
+		SecretsData:              secretsData,
 		FetchAndExtractTimeout:   c.cfg.FetchAndExtractTimeout,
 	}
 	log.Info("Dispatching image build", "images", buildOpts.Images)