diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 20bb11d6..5cdc0cf6 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -3,12 +3,18 @@ FROM registry.suse.com/bci/golang:1.19 ARG DAPPER_HOST_ARCH ENV ARCH $DAPPER_HOST_ARCH +ENV HELM_VERSION v3.12.1 +ENV HELM_UNITTEST_VERSION 0.3.2 + RUN zypper -n install git docker vim less file curl wget awk + +RUN curl -sL https://get.helm.sh/helm-${HELM_VERSION}-linux-${ARCH}.tar.gz | tar xvzf - -C /usr/local/bin --strip-components=1 + RUN if [ "${ARCH}" = "amd64" ]; then \ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.52.2; \ + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version ${HELM_UNITTEST_VERSION}>/out.txt 2>&1; \ fi -ENV HELM_VERSION v3.12.1 -RUN curl -sL https://get.helm.sh/helm-${HELM_VERSION}-linux-${ARCH}.tar.gz | tar xvzf - -C /usr/local/bin --strip-components=1 + RUN GOBIN=/usr/local/bin go install github.com/golang/mock/mockgen@v1.6.0 ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS diff --git a/charts/rancher-webhook/charts/capi/templates/service.yaml b/charts/rancher-webhook/charts/capi/templates/service.yaml index 08df65d6..de7c255c 100644 --- a/charts/rancher-webhook/charts/capi/templates/service.yaml +++ b/charts/rancher-webhook/charts/capi/templates/service.yaml @@ -8,6 +8,6 @@ spec: ports: - name: https port: 443 - targetPort: 8777 + targetPort: {{ .Values.port | default 8777 }} selector: app: rancher-webhook diff --git a/charts/rancher-webhook/templates/deployment.yaml b/charts/rancher-webhook/templates/deployment.yaml index 0c9776a2..13738fea 100644 --- a/charts/rancher-webhook/templates/deployment.yaml +++ b/charts/rancher-webhook/templates/deployment.yaml @@ -36,6 +36,10 @@ spec: value: "{{.Values.capi.enabled}}" - name: ENABLE_MCM value: "{{.Values.mcm.enabled}}" + - name: CATTLE_PORT + value: {{.Values.port | default 9443 | quote}} + - name: CATTLE_CAPI_PORT + value: {{.Values.capi.port | default 8777 | quote}} - name: NAMESPACE valueFrom: fieldRef: @@ -45,9 +49,9 @@ spec: imagePullPolicy: "{{ .Values.image.imagePullPolicy }}" ports: - name: https - containerPort: 9443 + containerPort: {{ .Values.port | default 9443 }} - name: capi-https - containerPort: 8777 + containerPort: {{ .Values.capi.port | default 8777}} startupProbe: httpGet: path: "/healthz" @@ -66,7 +70,14 @@ spec: - name: tls mountPath: /tmp/k8s-webhook-server/serving-certs {{- end }} + {{- if .Values.capNetBindService }} + securityContext: + capabilities: + add: + - NET_BIND_SERVICE + {{- end }} serviceAccountName: rancher-webhook {{- if .Values.priorityClassName }} priorityClassName: "{{.Values.priorityClassName}}" {{- end }} + \ No newline at end of file diff --git a/charts/rancher-webhook/templates/service.yaml b/charts/rancher-webhook/templates/service.yaml index 74a8a9e5..220afebe 100644 --- a/charts/rancher-webhook/templates/service.yaml +++ b/charts/rancher-webhook/templates/service.yaml @@ -6,7 +6,7 @@ metadata: spec: ports: - port: 443 - targetPort: 9443 + targetPort: {{ .Values.port | default 9443 }} protocol: TCP name: https selector: diff --git a/charts/rancher-webhook/tests/README.md b/charts/rancher-webhook/tests/README.md new file mode 100644 index 00000000..6d3059a0 --- /dev/null +++ b/charts/rancher-webhook/tests/README.md @@ -0,0 +1,16 @@ + +## local dev testing instructions + +Option 1: Full chart CI run with a live cluster + +```bash +./scripts/charts/ci +``` + +Option 2: Test runs against the chart only + +```bash +# install the helm plugin first - helm plugin install https://github.com/helm-unittest/helm-unittest.git +bash dev-scripts/helm-unittest.sh +``` + diff --git a/charts/rancher-webhook/tests/capi-service_test.yaml b/charts/rancher-webhook/tests/capi-service_test.yaml new file mode 100644 index 00000000..4ee94a84 --- /dev/null +++ b/charts/rancher-webhook/tests/capi-service_test.yaml @@ -0,0 +1,20 @@ +suite: Test Service +templates: + - charts/capi/templates/service.yaml +tests: + - it: should set webhook default port values + set: + capi.enabled: true + asserts: + - equal: + path: spec.ports[0].targetPort + value: 8777 + + - it: should set updated target port + set: + capi.port: 2319 + capi.enabled: true + asserts: + - equal: + path: spec.ports[0].targetPort + value: 2319 diff --git a/charts/rancher-webhook/tests/deployment_test.yaml b/charts/rancher-webhook/tests/deployment_test.yaml new file mode 100644 index 00000000..66a74d4e --- /dev/null +++ b/charts/rancher-webhook/tests/deployment_test.yaml @@ -0,0 +1,62 @@ +suite: Test Deployment +templates: + - deployment.yaml + +tests: + - it: should set webhook default port values + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 9443 + - equal: + path: spec.template.spec.containers[0].ports[1].containerPort + value: 8777 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_PORT + value: "9443" + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_CAPI_PORT + value: "8777" + + - it: should set updated webhook port + set: + port: 2319 + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 2319 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_PORT + value: "2319" + + - it: should set updated capi port + set: + capi.port: 2319 + asserts: + - equal: + path: spec.template.spec.containers[0].ports[1].containerPort + value: 2319 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_CAPI_PORT + value: "2319" + + - it: should not set capabilities by default. + asserts: + - isNull: + path: spec.template.spec.containers[0].securityContext + + - it: should set net capabilities when capNetBindService is true. + set: + capNetBindService: true + asserts: + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.add + content: NET_BIND_SERVICE diff --git a/charts/rancher-webhook/tests/service_test.yaml b/charts/rancher-webhook/tests/service_test.yaml new file mode 100644 index 00000000..03172ad0 --- /dev/null +++ b/charts/rancher-webhook/tests/service_test.yaml @@ -0,0 +1,18 @@ +suite: Test Service +templates: + - service.yaml + +tests: + - it: should set webhook default port values + asserts: + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: should set updated target port + set: + port: 2319 + asserts: + - equal: + path: spec.ports[0].targetPort + value: 2319 diff --git a/charts/rancher-webhook/values.yaml b/charts/rancher-webhook/values.yaml index 71e8f85d..9c1e1aeb 100644 --- a/charts/rancher-webhook/values.yaml +++ b/charts/rancher-webhook/values.yaml @@ -10,6 +10,7 @@ global: capi: enabled: false + port: 8777 mcm: enabled: true @@ -20,3 +21,6 @@ nodeSelector: {} ## PriorityClassName assigned to deployment. priorityClassName: "" + +# port assigns which port to use when running rancher-webhook +port: 9443 diff --git a/dev-scripts/helm-unittest.sh b/dev-scripts/helm-unittest.sh new file mode 100755 index 00000000..bb67864a --- /dev/null +++ b/dev-scripts/helm-unittest.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd $(dirname $0)/.. +./scripts/package-helm +./scripts/test-helm diff --git a/pkg/capi/capi.go b/pkg/capi/capi.go index a7c07790..1ddc684a 100644 --- a/pkg/capi/capi.go +++ b/pkg/capi/capi.go @@ -4,8 +4,10 @@ package capi import ( "context" "crypto/tls" + "fmt" "os" "path/filepath" + "strconv" controllerruntime "github.com/rancher/lasso/controller-runtime" "github.com/rancher/webhook/pkg/clients" @@ -43,13 +45,23 @@ func init() { _ = apiextensionsv1.AddToScheme(schemes.All) } -var ( - tlsCert = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs", "tls.crt") - capiPort = 8777 +const ( + defaultCapiPort = 8777 + capiPortEnvKey = "CATTLE_CAPI_PORT" ) +var tlsCert = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs", "tls.crt") + // Register registers a new CAPI webhook server and returns a start function. func Register(clients *clients.Clients, tlsOpts ...func(*tls.Config)) (func(ctx context.Context) error, error) { + capiPort := defaultCapiPort + if portStr := os.Getenv(capiPortEnvKey); portStr != "" { + var err error + capiPort, err = strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("failed to decode CAPI port value '%s': %w", portStr, err) + } + } mgr, err := ctrl.NewManager(clients.RESTConfig, ctrl.Options{ MetricsBindAddress: "0", NewCache: controllerruntime.NewNewCacheFunc(clients.SharedControllerFactory.SharedCacheFactory(), diff --git a/pkg/server/server.go b/pkg/server/server.go index 8d847b7a..455dfce2 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "fmt" "os" + "strconv" "time" "github.com/gorilla/mux" @@ -25,20 +26,18 @@ import ( ) const ( - serviceName = "rancher-webhook" - namespace = "cattle-system" - tlsName = "rancher-webhook.cattle-system.svc" - certName = "cattle-webhook-tls" - caName = "cattle-webhook-ca" - webhookHTTPPort = 0 // value of 0 indicates we do not want to use http. - webhookHTTPSPort = 9443 -) - -var ( - // These strings have to remain as vars since we need the address below. - validationPath = "/v1/webhook/validation" - mutationPath = "/v1/webhook/mutation" - clientPort = int32(443) + serviceName = "rancher-webhook" + namespace = "cattle-system" + tlsName = "rancher-webhook.cattle-system.svc" + certName = "cattle-webhook-tls" + caName = "cattle-webhook-ca" + validationPath = "/v1/webhook/validation" + mutationPath = "/v1/webhook/mutation" + clientPort = int32(443) + webhookHTTPPort = 0 // value of 0 indicates we do not want to use http. + defaultWebhookHTTPSPort = 9443 + webhookPortEnvKey = "CATTLE_PORT" + webhookURLEnvKey = "CATTLE_WEBHOOK_URL" ) // tlsOpt option function applied to all webhook servers. @@ -150,7 +149,14 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators [] tlsConfig := &tls.Config{} tlsOpt(tlsConfig) - + webhookHTTPSPort := defaultWebhookHTTPSPort + if portStr := os.Getenv(webhookPortEnvKey); portStr != "" { + var err error + webhookHTTPSPort, err = strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("failed to decode webhook port value '%s': %w", portStr, err) + } + } return server.ListenAndServe(ctx, webhookHTTPSPort, webhookHTTPPort, router, &server.ListenOpts{ Secrets: clients.Core.Secret(), CertNamespace: namespace, @@ -188,8 +194,8 @@ func (s *secretHandler) sync(_ string, secret *corev1.Secret) (*corev1.Secret, e Service: &v1.ServiceReference{ Namespace: namespace, Name: serviceName, - Path: &validationPath, - Port: &clientPort, + Path: admission.Ptr(validationPath), + Port: admission.Ptr(clientPort), }, CABundle: secret.Data[corev1.TLSCertKey], } @@ -198,12 +204,12 @@ func (s *secretHandler) sync(_ string, secret *corev1.Secret) (*corev1.Secret, e Service: &v1.ServiceReference{ Namespace: namespace, Name: serviceName, - Path: &mutationPath, - Port: &clientPort, + Path: admission.Ptr(mutationPath), + Port: admission.Ptr(clientPort), }, CABundle: secret.Data[corev1.TLSCertKey], } - if devURL, ok := os.LookupEnv("CATTLE_WEBHOOK_URL"); ok { + if devURL, ok := os.LookupEnv(webhookURLEnvKey); ok { validationURL := devURL + validationPath mutationURL := devURL + mutationPath validationClientConfig = v1.WebhookClientConfig{ diff --git a/scripts/ci b/scripts/ci index 52334105..51fe8e96 100755 --- a/scripts/ci +++ b/scripts/ci @@ -8,3 +8,4 @@ cd $(dirname $0) ./validate ./validate-ci ./package +./test-helm diff --git a/scripts/integration-test b/scripts/integration-test index 07ca1bec..483fef97 100755 --- a/scripts/integration-test +++ b/scripts/integration-test @@ -10,6 +10,7 @@ cd $(dirname $0)/../ echo "Starting Rancher Server" entrypoint.sh >./rancher.log 2>&1 & +RANCHER_PID=$! echo "Waiting for Rancher health check..." while ! curl -sf http://localhost:80/healthz >/dev/null 2>&1; do @@ -27,10 +28,37 @@ while ! kubectl rollout status -w -n cattle-system deploy/rancher-webhook >/dev/ echo "Waiting for rancher to deploy rancher-webhook..." sleep 2 done +echo "Webhook deployed" # After rancher deploys webhook kill the bash command running tail. kill ${TAIL_PID} +# Wait for helm operation to complete and save rancher-webhook release info before we kill rancher and the cluster. +while + pods=$(kubectl get pods -n cattle-system --field-selector=status.phase=Running -o jsonpath='{.items[?(@.metadata.generateName=="helm-operation-")].metadata.name}') + [[ -n "$pods" ]] +do + echo "Waiting for helm operation to finish" + sleep 2 +done + +# Kill Rancher since we only need the CRDs and the initial webhook values. +# We do not want Rancher to reconcile an older version of the webhook during test. +kill ${RANCHER_PID} + +echo "Rancher has been stopped starting K3s." +# Start Cluster without Rancher. +k3s server --cluster-init --disable=traefik,servicelb,metrics-server,local-storage --node-name=local-node --log=./k3s.log >/dev/null 2>&1 & +KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +# Wait for cluster to start. +while ! kubectl version >/dev/null 2>&1; do + echo "Waiting for cluster to start" + sleep 5 +done + +echo "Uploading new webhook image" + ###### Upload the newly created webhook image to containerd, then install the webhook chart using the new image IMAGE_FILE=./dist/rancher-webhook-image.tar # import image to containerd and get the image name @@ -40,15 +68,28 @@ WEBHOOK_REPO=$(ctr image import ${IMAGE_FILE} | cut -d ' ' -f 2 | cut -d ':' -f source ./dist/tags # Install the webhook chart we just built. -helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system --set image.repository=${WEBHOOK_REPO} --set image.tag=${TAG} --reuse-values - -while ! kubectl rollout status -w -n cattle-system deploy/rancher-webhook; do +# This command can fail since it is so close to the cluster start so we will give it 3 retires. +RETRIES=0 +while ! helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ + --wait --set image.repository=${WEBHOOK_REPO} --set image.tag=${TAG} --reuse-values; do + if [ "$RETRIES" -ge 3 ]; then + exit 1 + fi + RETRIES=$((RETRIES + 1)) sleep 2 done ./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest -# Scale down rancher-webhook so that we can run tests on the FailurePolicy +# Install the webhook chart with new ports. +helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ + --wait --reuse-values --set port=443 --set capi.port=2319 + +# Test that the ports are set as expected and run a single integration test to verify the webhook is still accessible. +./bin/rancher-webhook-integration.test -test.v -test.run PortTest +./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest -testify.m TestGlobalRole + +# Scale down rancher-webhook so that we can run tests on the FailurePolicy. kubectl scale deploy rancher-webhook -n cattle-system --replicas=0 kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system ./bin/rancher-webhook-integration.test -test.v -test.run FailurePolicyTest diff --git a/scripts/package-helm b/scripts/package-helm index 394c9af4..45d5c4e2 100755 --- a/scripts/package-helm +++ b/scripts/package-helm @@ -12,13 +12,16 @@ rm -rf build/charts mkdir -p build dist/artifacts cp -rf charts build/ -sed -i \ +# must use sed -i''` for GNU and OSX compatibility +sed -i'.bkp' \ -e 's/^version:.*/version: '${HELM_VERSION}'/' \ -e 's/appVersion:.*/appVersion: '${HELM_VERSION}'/' \ build/charts/rancher-webhook/Chart.yaml -sed -i \ +sed -i'.bkb' \ -e 's/tag: latest/tag: '${HELM_TAG}'/' \ build/charts/rancher-webhook/values.yaml +rm build/charts/rancher-webhook/Chart.yaml.bkp build/charts/rancher-webhook/values.yaml.bkb + helm package -d ./dist/artifacts ./build/charts/rancher-webhook diff --git a/scripts/test-helm b/scripts/test-helm new file mode 100755 index 00000000..a1e95d1e --- /dev/null +++ b/scripts/test-helm @@ -0,0 +1,15 @@ +#! /bin/bash +set -e +cd $(dirname $0)/.. + +./scripts/package-helm +echo Running helm lint +helm lint ./charts/rancher-webhook +# Check for unittest plugin +if helm unittest --help >/dev/null 2>&1; then + helm unittest build/charts/rancher-webhook +else + echo "skipping helm unittest" + echo "helm plugin unittest not found." + echo "Run to install plugin: helm plugin install https://github.com/helm-unittest/helm-unittest.git" +fi diff --git a/scripts/validate-ci b/scripts/validate-ci index b025e93c..f768a0da 100755 --- a/scripts/validate-ci +++ b/scripts/validate-ci @@ -16,9 +16,6 @@ go mod verify source ./scripts/version -echo Running helm lint -helm lint ./charts/rancher-webhook - if [ -n "$DIRTY" ]; then echo Git is dirty git status diff --git a/tests/integration/port_test.go b/tests/integration/port_test.go new file mode 100644 index 00000000..c012907d --- /dev/null +++ b/tests/integration/port_test.go @@ -0,0 +1,81 @@ +package integration_test + +import ( + "context" + "os" + "testing" + + "github.com/rancher/lasso/pkg/client" + "github.com/rancher/wrangler/pkg/kubeconfig" + "github.com/rancher/wrangler/pkg/schemes" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + testWebhookPort = 443 + testCapiPort = 2319 +) + +type PortSuite struct { + suite.Suite + clientFactory client.SharedClientFactory +} + +// TestPortTest should be run only when the webhook is not running. +func TestPortTest(t *testing.T) { + suite.Run(t, new(PortSuite)) +} + +func (m *PortSuite) SetupSuite() { + logrus.SetLevel(logrus.DebugLevel) + kubeconfigPath := os.Getenv("KUBECONFIG") + restCfg, err := kubeconfig.GetNonInteractiveClientConfig(kubeconfigPath).ClientConfig() + m.Require().NoError(err, "Failed to clientFactory config") + m.clientFactory, err = client.NewSharedClientFactoryForConfig(restCfg) + m.Require().NoError(err, "Failed to create clientFactory Interface") + + schemes.Register(corev1.AddToScheme) +} + +func (m *PortSuite) TestWebhookPortChanged() { + podGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + } + + podClient, err := m.clientFactory.ForKind(podGVK) + m.Require().NoError(err, "Failed to create client") + listOpts := v1.ListOptions{ + LabelSelector: "app=rancher-webhook", + } + pods := corev1.PodList{} + podClient.List(context.Background(), "cattle-system", &pods, listOpts) + var webhookPod *corev1.Pod + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + if webhookPod != nil { + m.Require().FailNow("more then one rancher-webhook pod is running") + } + webhookPod = &pod + } + } + if webhookPod == nil { + m.Require().FailNow("running webhook pod not found") + } + m.Require().Equal(corev1.PodRunning, webhookPod.Status.Phase, "Rancher-webhook pod is not running Phase=%s", webhookPod.Status.Phase) + m.Require().Len(webhookPod.Spec.Containers, 1, "Rancher-webhook pod has the incorrect number of containers") + m.Require().Len(webhookPod.Spec.Containers[0].Ports, 2, "Rancher-webhook container has the incorrect number of ports") + havePort1 := webhookPod.Spec.Containers[0].Ports[0].ContainerPort + havePort2 := webhookPod.Spec.Containers[0].Ports[1].ContainerPort + if havePort1 != testWebhookPort && havePort2 != testWebhookPort { + m.Require().FailNowf("expected webhook port not found", "wanted '%d' was not found instead have '%d' and '%d'", testWebhookPort, havePort1, havePort2) + } + if havePort1 != testCapiPort && havePort2 != testCapiPort { + m.Require().FailNowf("expected capi port not found", "wanted '%d' was not found instead have '%d' and '%d'", testCapiPort, havePort1, havePort2) + } +}