Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

First take at getting spintainer working #278

Merged
merged 1 commit into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions api/v1alpha1/spinappexecutor_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,15 @@ type SpinAppExecutorSpec struct {

type ExecutorDeploymentConfig struct {
// RuntimeClassName is the runtime class name that should be used by pods created
// as part of a deployment.
RuntimeClassName string `json:"runtimeClassName"`
// as part of a deployment. This should only be defined when SpintainerImage is not defined.
RuntimeClassName *string `json:"runtimeClassName,omitempty"`

// SpinImage points to an image that will run Spin in a container to execute
// your SpinApp. This is an alternative to using the shim to execute your
// SpinApp. This should only be defined when RuntimeClassName is not
// defined. When specified, application images must be available without
// authentication.
SpinImage *string `json:"spinImage,omitempty"`

// CACertSecret specifies the name of the secret containing the CA
// certificates to be mounted to the deployment.
Expand Down
10 changes: 10 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 9 additions & 3 deletions config/crd/bases/core.spinoperator.dev_spinappexecutors.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,16 @@ spec:
runtimeClassName:
description: |-
RuntimeClassName is the runtime class name that should be used by pods created
as part of a deployment.
as part of a deployment. This should only be defined when SpintainerImage is not defined.
type: string
spinImage:
description: |-
SpinImage points to an image that will run Spin in a container to execute
your SpinApp. This is an alternative to using the shim to execute your
SpinApp. This should only be defined when RuntimeClassName is not
defined. When specified, application images must be available without
authentication.
type: string
required:
- runtimeClassName
type: object
required:
- createDeployment
Expand Down
9 changes: 9 additions & 0 deletions config/samples/spintainer-executor.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: core.spinoperator.dev/v1alpha1
kind: SpinAppExecutor
metadata:
name: spintainer
spec:
createDeployment: true
deploymentConfig:
installDefaultCACerts: true
spinImage: ghcr.io/fermyon/spin:v2.7.0
8 changes: 8 additions & 0 deletions config/samples/spintainer.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: core.spinoperator.dev/v1alpha1
kind: SpinApp
metadata:
name: spintainer-spinapp
spec:
image: "ghcr.io/spinkube/spin-operator/hello-world:20240708-130250-gfefd2b1"
replicas: 1
executor: spintainer
4 changes: 2 additions & 2 deletions e2e/crd_installed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func TestCRDInstalled(t *testing.T) {
Assess("spinapp crd installed", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
client := cfg.Client()
if err := apiextensionsV1.AddToScheme(client.Resources().GetScheme()); err != nil {
t.Fatalf("failed to register the v1 API extension types with Kuberenets scheme: %s", err)
t.Fatalf("failed to register the v1 API extension types with Kubernetes scheme: %s", err)
}
name := "spinapps.core.spinoperator.dev"
var crd apiextensionsV1.CustomResourceDefinition
Expand All @@ -31,7 +31,7 @@ func TestCRDInstalled(t *testing.T) {
Assess("spinappexecutor crd installed", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
client := cfg.Client()
if err := apiextensionsV1.AddToScheme(client.Resources().GetScheme()); err != nil {
t.Fatalf("failed to register the v1 API extension types with Kuberenets scheme: %s", err)
t.Fatalf("failed to register the v1 API extension types with Kubernetes scheme: %s", err)
}

name := "spinappexecutors.core.spinoperator.dev"
Expand Down
6 changes: 3 additions & 3 deletions e2e/default_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestDefaultSetup(t *testing.T) {
Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
client = cfg.Client()

testSpinApp := newSpinAppCR(testSpinAppName, helloWorldImage)
testSpinApp := newSpinAppCR(testSpinAppName, helloWorldImage, "containerd-shim-spin")
if err := client.Resources().Create(ctx, testSpinApp); err != nil {
t.Fatalf("Failed to create spinapp: %s", err)
}
Expand Down Expand Up @@ -69,7 +69,7 @@ func TestDefaultSetup(t *testing.T) {
testEnv.Test(t, defaultTest)
}

func newSpinAppCR(name, image string) *spinapps_v1alpha1.SpinApp {
func newSpinAppCR(name, image, executor string) *spinapps_v1alpha1.SpinApp {
return &spinapps_v1alpha1.SpinApp{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -78,7 +78,7 @@ func newSpinAppCR(name, image string) *spinapps_v1alpha1.SpinApp {
Spec: spinapps_v1alpha1.SpinAppSpec{
Replicas: 1,
Image: image,
Executor: "containerd-shim-spin",
Executor: executor,
},
}
}
2 changes: 1 addition & 1 deletion e2e/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func newContainerdShimExecutor(namespace string) *spinapps_v1alpha1.SpinAppExecu
Spec: spinapps_v1alpha1.SpinAppExecutorSpec{
CreateDeployment: true,
DeploymentConfig: &spinapps_v1alpha1.ExecutorDeploymentConfig{
RuntimeClassName: runtimeClassName,
RuntimeClassName: &runtimeClassName,
InstallDefaultCACerts: true,
CACertSecret: testCACertSecret,
},
Expand Down
107 changes: 107 additions & 0 deletions e2e/spintainer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package e2e

import (
"context"
"testing"
"time"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/e2e-framework/klient"
"sigs.k8s.io/e2e-framework/klient/k8s"
"sigs.k8s.io/e2e-framework/klient/wait"
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/features"

spinapps_v1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1"
"github.com/spinkube/spin-operator/internal/generics"
)

// TestSpintainer is a test that checks that the minimal setup works
// with the spintainer executor
func TestSpintainer(t *testing.T) {
var client klient.Client

helloWorldImage := "ghcr.io/spinkube/spin-operator/hello-world:20240708-130250-gfefd2b1"
testSpinAppName := "test-spintainer-app"

defaultTest := features.New("default and most minimal setup").
Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {

client = cfg.Client()

if err := spinapps_v1alpha1.AddToScheme(client.Resources(testNamespace).GetScheme()); err != nil {
t.Fatalf("failed to register the spinapps_v1alpha1 types with Kubernetes scheme: %s", err)
}

return ctx
}).
Assess("spin app custom resource is created", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
testSpinApp := newSpinAppCR(testSpinAppName, helloWorldImage, "spintainer")

if err := client.Resources().Create(ctx, newSpintainerExecutor(testNamespace)); err != nil {
t.Fatalf("Failed to create spinappexecutor: %s", err)
}

if err := client.Resources().Create(ctx, testSpinApp); err != nil {
t.Fatalf("Failed to create spinapp: %s", err)
}
// wait for spinapp to be created
if err := wait.For(
conditions.New(client.Resources()).ResourceMatch(testSpinApp, func(object k8s.Object) bool {
return true
}),
wait.WithTimeout(3*time.Minute),
wait.WithInterval(30*time.Second),
); err != nil {
t.Fatal(err)
}

return ctx
}).
Assess("spin app deployment and service are available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
// wait for deployment to be ready
if err := wait.For(
conditions.New(client.Resources()).DeploymentAvailable(testSpinAppName, testNamespace),
wait.WithTimeout(3*time.Minute),
wait.WithInterval(30*time.Second),
); err != nil {
t.Fatal(err)
}

svc := &v1.ServiceList{
Items: []v1.Service{
{ObjectMeta: metav1.ObjectMeta{Name: testSpinAppName, Namespace: testNamespace}},
},
}

if err := wait.For(
conditions.New(client.Resources()).ResourcesFound(svc),
wait.WithTimeout(3*time.Minute),
wait.WithInterval(30*time.Second),
); err != nil {
t.Fatal(err)
}
return ctx
}).
Feature()
testEnv.Test(t, defaultTest)
}

func newSpintainerExecutor(namespace string) *spinapps_v1alpha1.SpinAppExecutor {
var testSpinAppExecutor = &spinapps_v1alpha1.SpinAppExecutor{
ObjectMeta: metav1.ObjectMeta{
Name: "spintainer",
Namespace: namespace,
},
Spec: spinapps_v1alpha1.SpinAppExecutorSpec{
CreateDeployment: true,
DeploymentConfig: &spinapps_v1alpha1.ExecutorDeploymentConfig{
SpinImage: generics.Ptr("ghcr.io/fermyon/spin:v2.7.0"),
},
},
}

return testSpinAppExecutor
}
4 changes: 2 additions & 2 deletions internal/controller/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func constructRuntimeConfigSecretMount(_ctx context.Context, secretName string)
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
Optional: ptr(true),
Optional: generics.Ptr(true),
Items: []corev1.KeyToPath{
{
Key: "runtime-config.toml",
Expand All @@ -46,7 +46,7 @@ func constructCASecretMount(_ context.Context, caSecretName string) (corev1.Volu
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: caSecretName,
Optional: ptr(true),
Optional: generics.Ptr(true),
Items: []corev1.KeyToPath{{
Key: "ca-certificates.crt",
Path: "ca-certificates.crt",
Expand Down
12 changes: 11 additions & 1 deletion internal/controller/deployment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
clientgoscheme "k8s.io/client-go/kubernetes/scheme"

spinv1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1"
"github.com/spinkube/spin-operator/internal/generics"
"github.com/spinkube/spin-operator/pkg/spinapp"
)

Expand Down Expand Up @@ -280,7 +281,16 @@ func TestSpinHealthCheckToCoreProbe(t *testing.T) {
func TestDeploymentLabel(t *testing.T) {
scheme := registerAndGetScheme()
app := minimalSpinApp()
deployment, err := constructDeployment(context.Background(), app, &spinv1alpha1.ExecutorDeploymentConfig{}, "", "", scheme)
deployment, err := constructDeployment(
context.Background(),
app,
&spinv1alpha1.ExecutorDeploymentConfig{
RuntimeClassName: generics.Ptr("containerd-shim-spin"),
},
"",
"",
scheme,
)

require.Nil(t, err)
require.NotNil(t, deployment.ObjectMeta.Labels)
Expand Down
66 changes: 42 additions & 24 deletions internal/controller/spinapp_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controller

import (
"context"
"errors"
"fmt"
"hash/adler32"
"maps"
Expand All @@ -37,6 +38,7 @@ import (

spinv1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1"
"github.com/spinkube/spin-operator/internal/cacerts"
"github.com/spinkube/spin-operator/internal/generics"
"github.com/spinkube/spin-operator/internal/logging"
"github.com/spinkube/spin-operator/internal/runtimeconfig"
"github.com/spinkube/spin-operator/pkg/spinapp"
Expand Down Expand Up @@ -326,7 +328,7 @@ func (r *SpinAppReconciler) reconcileDeployment(ctx context.Context, app *spinv1
// We want to use server-side apply https://kubernetes.io/docs/reference/using-api/server-side-apply
patchMethod := client.Apply
patchOptions := &client.PatchOptions{
Force: ptr(true), // Force b/c any fields we are setting need to be owned by the spin-operator
Force: generics.Ptr(true), // Force b/c any fields we are setting need to be owned by the spin-operator
FieldManager: FieldManager,
}

Expand Down Expand Up @@ -354,7 +356,7 @@ func (r *SpinAppReconciler) reconcileService(ctx context.Context, app *spinv1alp
// We want to use server-side apply https://kubernetes.io/docs/reference/using-api/server-side-apply
patchMethod := client.Apply
patchOptions := &client.PatchOptions{
Force: ptr(true), // Force b/c any fields we are setting need to be owned by the spin-operator
Force: generics.Ptr(true), // Force b/c any fields we are setting need to be owned by the spin-operator
FieldManager: FieldManager,
}
// Note that we reconcile even if the service is in a good state. We rely on controller-runtime to rate limit us.
Expand Down Expand Up @@ -390,7 +392,7 @@ func constructDeployment(ctx context.Context, app *spinv1alpha1.SpinApp, config
if app.Spec.EnableAutoscaling {
replicas = nil
} else {
replicas = ptr(app.Spec.Replicas)
replicas = generics.Ptr(app.Spec.Replicas)
}

volumes, volumeMounts, err := ConstructVolumeMountsForApp(ctx, app, generatedRuntimeConfigSecretName, caSecretName)
Expand Down Expand Up @@ -435,6 +437,41 @@ func constructDeployment(ctx context.Context, app *spinv1alpha1.SpinApp, config

labels := constructAppLabels(app)

var container corev1.Container
if config.RuntimeClassName != nil {
container = corev1.Container{
Name: app.Name,
Image: app.Spec.Image,
Command: []string{"/"},
Ports: []corev1.ContainerPort{{
Name: spinapp.HTTPPortName,
ContainerPort: spinapp.DefaultHTTPPort,
}},
Env: env,
VolumeMounts: volumeMounts,
Resources: resources,
LivenessProbe: livenessProbe,
ReadinessProbe: readinessProbe,
}
} else if config.SpinImage != nil {
container = corev1.Container{
Name: app.Name,
Image: *config.SpinImage,
Args: []string{"up", "--listen", fmt.Sprintf("0.0.0.0:%d", spinapp.DefaultHTTPPort), "-f", app.Spec.Image},
calebschoepp marked this conversation as resolved.
Show resolved Hide resolved
Ports: []corev1.ContainerPort{{
Name: spinapp.HTTPPortName,
ContainerPort: spinapp.DefaultHTTPPort,
}},
Env: env,
VolumeMounts: volumeMounts,
Resources: resources,
LivenessProbe: livenessProbe,
ReadinessProbe: readinessProbe,
}
} else {
return nil, errors.New("must specify either runtimeClassName or spinImage")
}

dep := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
Expand All @@ -457,23 +494,8 @@ func constructDeployment(ctx context.Context, app *spinv1alpha1.SpinApp, config
Annotations: templateAnnotations,
},
Spec: corev1.PodSpec{
RuntimeClassName: &config.RuntimeClassName,
Containers: []corev1.Container{
{
Name: app.Name,
Image: app.Spec.Image,
Command: []string{"/"},
Ports: []corev1.ContainerPort{{
Name: spinapp.HTTPPortName,
ContainerPort: spinapp.DefaultHTTPPort,
}},
Env: env,
VolumeMounts: volumeMounts,
Resources: resources,
LivenessProbe: livenessProbe,
ReadinessProbe: readinessProbe,
},
},
RuntimeClassName: config.RuntimeClassName,
Containers: []corev1.Container{container},
ImagePullSecrets: app.Spec.ImagePullSecrets,
Volumes: volumes,
},
Expand Down Expand Up @@ -503,7 +525,3 @@ func (r *SpinAppReconciler) findDeploymentForApp(ctx context.Context, app *spinv
}
return &deployment, nil
}

func ptr[T any](v T) *T {
return &v
}
Loading
Loading