feat(lifecycle-operator): introduce option to enable lifecycle orches…

…tration only for specific namespaces (keptn#2244) Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com> Signed-off-by: odubajDT <93584209+odubajDT@users.noreply.github.com> Co-authored-by: Meg McRoberts <meg.mcroberts@dynatrace.com>
geoffrey1330 · Oct 11, 2023 · 12caf03 · 12caf03
1 parent c2cc89a
commit 12caf03
Show file tree

Hide file tree

Showing 23 changed files with 277 additions and 32 deletions.
diff --git a/.github/actions/deploy-klt-on-cluster/action.yml b/.github/actions/deploy-klt-on-cluster/action.yml
@@ -21,11 +21,15 @@ inputs:
   helm-install:
     required: false
     description: "Install Keptn via helm instead of manifest if true"
-    default: "true"
+    default: "helm_on"
   scheduling-gates:
     required: false
     description: "Use scheduling gates instead of scheduler"
-    default: "false"
+    default: "gates_off"
+  allowed-namespaces:
+    required: false
+    description: "Decides whether to allow only certain namespaces"
+    default: "allowed_ns_off"
 runs:
   using: "composite"
   steps:
@@ -61,7 +65,7 @@ runs:
         done
 
     - name: Install lifecycle-toolkit with manifests
-      if: inputs.helm-install == 'false' && inputs.scheduling-gates == 'false'
+      if: ${{  inputs.helm-install == 'helm_off' }}
       shell: bash
       run: |
         echo "Installing Keptn using manifests"
@@ -87,7 +91,7 @@ runs:
         kubectl rollout status deployment lifecycle-operator -n keptn-lifecycle-toolkit-system -w
 
     - name: Install lifecycle-toolkit with helm
-      if: inputs.helm-install == 'true'
+      if: ${{ inputs.helm-install == 'helm_on' }}
       env:
         RELEASE_REGISTRY: "localhost:5000/keptn"
       shell: bash
@@ -99,8 +103,21 @@ runs:
         helm dependency update
         helm dependency build
 
+        touch tmp-values.yaml
+        if [ "${{ inputs.allowed-namespaces }}" == "allowed_ns_on" ]; then
+          echo "lifecycleOperator:" >> tmp-values.yaml
+          echo "  allowedNamespaces: [allowed-ns-test]" >> tmp-values.yaml
+        fi
+
+        if [ "${{ inputs.scheduling-gates }}" == "gates_on" ]; then
+          if [ "${{ inputs.allowed-namespaces }}" == "allowed_ns_off" ]; then
+            echo "lifecycleOperator:" >> tmp-values.yaml
+          fi
+          echo "  schedulingGatesEnabled: true" >> tmp-values.yaml
+        fi
+
         helm install -n keptn-lifecycle-toolkit-system --create-namespace keptn ./ \
-          --set lifecycleOperator.schedulingGatesEnabled=${{ inputs.scheduling-gates }} \
+          --values tmp-values.yaml \
           --set lifecycleOperator.scheduler.imagePullPolicy=Never \
           --set lifecycleOperator.scheduler.image.tag=${{ inputs.runtime_tag }} \
           --set lifecycleOperator.scheduler.image.repository="localhost:5000/keptn/scheduler" \

diff --git a/.github/scripts/.helm-tests/default/result.yaml b/.github/scripts/.helm-tests/default/result.yaml
@@ -8611,9 +8611,6 @@ webhooks:
     - key: kubernetes.io/metadata.name
       operator: NotIn
       values:
-      - kube-system
-      - kube-public
-      - kube-node-lease
       - cert-manager
       - keptn-lifecycle-toolkit-system
       - observability
@@ -8622,6 +8619,9 @@ webhooks:
       operator: NotIn
       values:
       - 'helmtests'
+      - kube-system
+      - kube-public
+      - kube-node-lease
   rules:
   - apiGroups:
     - ""

diff --git a/.github/scripts/.helm-tests/lifecycle-only/result.yaml b/.github/scripts/.helm-tests/lifecycle-only/result.yaml
@@ -6697,9 +6697,6 @@ webhooks:
     - key: kubernetes.io/metadata.name
       operator: NotIn
       values:
-      - kube-system
-      - kube-public
-      - kube-node-lease
       - cert-manager
       - keptn-lifecycle-toolkit-system
       - observability
@@ -6708,6 +6705,9 @@ webhooks:
       operator: NotIn
       values:
       - 'helmtests'
+      - kube-system
+      - kube-public
+      - kube-node-lease
   rules:
   - apiGroups:
     - ""

diff --git a/.github/scripts/.helm-tests/lifecycle-with-certs/result.yaml b/.github/scripts/.helm-tests/lifecycle-with-certs/result.yaml
@@ -6953,9 +6953,6 @@ webhooks:
     - key: kubernetes.io/metadata.name
       operator: NotIn
       values:
-      - kube-system
-      - kube-public
-      - kube-node-lease
       - cert-manager
       - keptn-lifecycle-toolkit-system
       - observability
@@ -6964,6 +6961,9 @@ webhooks:
       operator: NotIn
       values:
       - 'helmtests'
+      - kube-system
+      - kube-public
+      - kube-node-lease
   rules:
   - apiGroups:
     - ""

diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
@@ -202,16 +202,24 @@ jobs:
       - build_image
     strategy:
       matrix:
-        helm: [true, false]
-        scheduling-gates: [true, false]
-        # do not run the tests with manifests installation and scheduling gates
+        helm: [helm_on, helm_off]
+        scheduling-gates: [gates_on, gates_off]
+        allowed-namespaces: [allowed_ns_on, allowed_ns_off]
         exclude:
-          - helm: false
-            scheduling-gates: true
+          - helm: helm_off
+            scheduling-gates: gates_on
+            allowed-namespaces: allowed_ns_on
+          - helm: helm_off
+            scheduling-gates: gates_off
+            allowed-namespaces: allowed_ns_on
+          - helm: helm_off
+            scheduling-gates: gates_on
+            allowed-namespaces: allowed_ns_off
     with:
       runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
       helm-install: ${{ matrix.helm }}
       scheduling-gates: ${{ matrix.scheduling-gates }}
+      allowed-namespaces: ${{ matrix.allowed-namespaces }}
     uses: ./.github/workflows/integration-test.yml
 
   load-tests:

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -8,12 +8,16 @@ on:
         required: true
       helm-install:
         description: "Decides whether to install via helm"
-        type: "boolean"
-        default: true
+        type: "string"
+        default: helm_on
       scheduling-gates:
-        description: "Decides whether to ue scheduling gates"
-        type: "boolean"
-        default: false
+        description: "Decides whether to use scheduling gates"
+        type: "string"
+        default: gates_off
+      allowed-namespaces:
+        description: "Decides whether to allow only certain namespaces"
+        type: "string"
+        default: allowed_ns_off
 env:
   GO_VERSION: "~1.20"
   # renovate: datasource=github-tags depName=kudobuilder/kuttl
@@ -36,6 +40,7 @@ jobs:
           runtime_tag: ${{ inputs.runtime_tag }}
           helm-install: ${{ inputs.helm-install }}
           scheduling-gates: ${{ inputs.scheduling-gates }}
+          allowed-namespaces: ${{ inputs.allowed-namespaces }}
 
       - name: Install and expose Prometheus
         uses: ./.github/actions/deploy-prometheus-on-cluster
@@ -49,11 +54,17 @@ jobs:
           mv kubectl-kuttl /usr/local/bin
 
       - name: Run Scheduling Gates Integration Tests
-        if: inputs.scheduling-gates == true
+        if: inputs.scheduling-gates == 'gates_on' && inputs.allowed-namespaces == 'allowed_ns_off'
         working-directory: .
         run: make integration-test-scheduling-gates
 
+      - name: Run Allowed namespaces Integration Tests
+        if: inputs.allowed-namespaces == 'allowed_ns_on'
+        working-directory: .
+        run: make integration-test-allowed-namespaces
+
       - name: Run Integration Tests
+        if: inputs.allowed-namespaces == 'allowed_ns_off'
         working-directory: .
         run: make integration-test
 
@@ -66,5 +77,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: logs-integration-tests-helm_${{ inputs.helm-install }}-schedule_${{ inputs.scheduling-gates }}
+          name: logs-integration-tests-${{ inputs.helm-install }}-${{ inputs.scheduling-gates }}-${{ inputs.allowed-namespaces }}
           path: .github/scripts/logs
diff --git a/Makefile b/Makefile
@@ -44,6 +44,14 @@ integration-test-scheduling-gates:	# to run a single test by name use --test eg.
 integration-test-scheduling-gates-local: install-prometheus
 	kubectl kuttl test --start-kind=false ./test/scheduling-gates/ --config=kuttl-test-local.yaml
 
+.PHONY: integration-test-allowed-namespaces #these tests should run on a real cluster!
+integration-test-allowed-namespaces:	# to run a single test by name use --test eg. --test=expose-keptn-metric
+	kubectl kuttl test --start-kind=false ./test/allowed-namespaces/ --config=kuttl-test.yaml
+
+.PHONY: integration-test-allowed-namespaces-local #these tests should run on a real cluster!
+integration-test-allowed-namespaces-local: install-prometheus
+	kubectl kuttl test --start-kind=false ./test/allowed-namespaces/ --config=kuttl-test-local.yaml
+
 .PHONY: load-test
 load-test:
 	kubectl apply -f ./test/load/assets/templates/namespace.yaml

diff --git a/README.md b/README.md
@@ -78,6 +78,52 @@ helm repo update
 helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --create-namespace --wait
 ```
 
+### Installation with only certain namespaces allowed
+
+Keptn lifecycle orchestration is by default enabled for all namespaces except the following ones:
+
+- `kube-system`
+- `kube-public`
+- `kube-node-lease`
+- `cert-manager`
+- `keptn-lifecycle-toolkit-system`
+- `observability`
+- `monitoring`
+- `<Keptn installation namespace>`
+
+To restrict Keptn lifecycle orchestration to specific namespaces, you must specify
+those namespaces during installation via helm values.
+First you need to create a `values.yaml`
+file
+
+```yaml
+lifecycleOperator:
+  allowedNamespaces:
+  - allowed-ns-1
+  - allowed-ns-2
+```
+
+and add the values file to the helm installation command:
+
+```shell
+helm repo add klt https://charts.lifecycle.keptn.sh
+helm repo update
+helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --values values.yaml --create-namespace --wait
+```
+
+> **Note**
+Please be aware that you still need to correctly annotate the namespaces where
+Keptn lifecycle orchestration is allowed.
+> To annotate them, use:
+
+```shell
+kubectl annotate ns <your-allowed-namespace> keptn.sh/lifecycle-toolkit='enabled'
+```
+
+> **Note**
+Please be aware that, if this option is set, adding any additional namespace
+requires the helm installation to be updated by adding the name of the new namespace to the list.
+
 ### Installation without scheduler
 
 Keptn installed on Kubernetes cluster running Kubernetes >= 1.26

diff --git a/lifecycle-operator/chart/README.md b/lifecycle-operator/chart/README.md
@@ -64,6 +64,7 @@ and application health checks
 | `kubernetesClusterDomain` | overrides domain.local                                                                                                                          | `cluster.local` |
 | `imagePullSecrets`        | global value for image registry secret                                                                                                          | `[]`            |
 | `schedulingGatesEnabled`  | enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version | `false`         |
+| `allowedNamespaces`       | specifies the allowed namespaces for the lifecycle orchestration functionality                                                                  | `[]`            |
 
 ### Keptn Scheduler
 

diff --git a/lifecycle-operator/chart/doc.yaml b/lifecycle-operator/chart/doc.yaml
@@ -77,6 +77,7 @@
 ## @param  kubernetesClusterDomain overrides domain.local
 ## @param  imagePullSecrets global value for image registry secret
 ## @param  schedulingGatesEnabled enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version
+## @param  allowedNamespaces specifies the allowed namespaces for the lifecycle orchestration functionality
 
 # yamllint disable rule:line-length
 ## @section Keptn Scheduler

diff --git a/lifecycle-operator/chart/templates/lifecycle-mutating-webhook-configuration.yaml b/lifecycle-operator/chart/templates/lifecycle-mutating-webhook-configuration.yaml
@@ -24,20 +24,26 @@ webhooks:
       operator: NotIn
       values:
       - lifecycle-operator
+{{- if eq (len .Values.allowedNamespaces) 0 }}
     - key: kubernetes.io/metadata.name
       operator: NotIn
       values:
-      - kube-system
-      - kube-public
-      - kube-node-lease
       - cert-manager
       - keptn-lifecycle-toolkit-system
       - observability
       - monitoring
+{{- else }}
+    - key: kubernetes.io/metadata.name
+      operator: In
+      values: {{ .Values.allowedNamespaces | default list | toJson }}
+{{- end }}
     - key: kubernetes.io/metadata.name
       operator: NotIn
       values:
       - '{{ .Release.Namespace }}'
+      - kube-system
+      - kube-public
+      - kube-node-lease
   rules:
   - apiGroups:
     - ""

diff --git a/lifecycle-operator/chart/values.yaml b/lifecycle-operator/chart/values.yaml
@@ -1,6 +1,7 @@
 imagePullSecrets: []
 kubernetesClusterDomain: cluster.local
 schedulingGatesEnabled: false
+allowedNamespaces: []
 lifecycleOperatorConfig:
   health:
     healthProbeBindAddress: :8081

diff --git a/test/allowed-namespaces/simple-deployment-allowed/00-assert.yaml b/test/allowed-namespaces/simple-deployment-allowed/00-assert.yaml
@@ -0,0 +1,9 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: test
+  name: test
+  namespace: allowed-ns-test
+status:
+  readyReplicas: 2
diff --git a/test/allowed-namespaces/simple-deployment-allowed/00-install.yaml b/test/allowed-namespaces/simple-deployment-allowed/00-install.yaml
@@ -0,0 +1,43 @@
+---
+apiVersion: lifecycle.keptn.sh/v1alpha3
+kind: KeptnTaskDefinition
+metadata:
+  name: pre-deployment-hello
+  namespace: allowed-ns-test
+spec:
+  function:
+    inline:
+      code: |
+        console.log("Pre-Deployment Task has been executed");
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: test
+  name: test
+  namespace: allowed-ns-test
+  annotations:
+    keptn.sh/workload: waiter
+    keptn.sh/version: "0.4"
+    keptn.sh/pre-deployment-tasks: pre-deployment-hello
+    keptn.sh/post-deployment-tasks: pre-deployment-hello
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: test
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: test
+    spec:
+      containers:
+        - image: busybox
+          name: busybox
+          command: ['sh', '-c', 'echo The app is running! && sleep infinity']
+      initContainers:
+        - name: init-myservice
+          image: busybox:1.36.1
+          command: ['sh', '-c', 'sleep 10']
diff --git a/test/allowed-namespaces/simple-deployment-allowed/00-teststep.yaml b/test/allowed-namespaces/simple-deployment-allowed/00-teststep.yaml
@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1
+kind: TestStep
+commands:
+  - script: kubectl create ns allowed-ns-test
+  - script: kubectl annotate ns allowed-ns-test keptn.sh/lifecycle-toolkit='enabled'