feat: add nodeSelector, pod tolerations, node affinity configuration

eshepelyuk · Nov 26, 2021 · 5733852 · 5733852
1 parent ef5cbbb
commit 5733852
Show file tree

Hide file tree

Showing 10 changed files with 219 additions and 5 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -21,7 +21,7 @@ jobs:
       - run: ./test/linter/test.sh
       - run: |
           helm plugin install https://github.com/quintush/helm-unittest
-          helm unittest -f test/unit/*.yaml -3 .
+          helm unittest -f 'test/unit/*.yaml' -3 .
       - uses: AbsaOSS/k3d-action@v1.5.0
         with:
           cluster-name: "k3d"

diff --git a/templates/cronjob-reconcile.yaml b/templates/cronjob-reconcile.yaml
@@ -25,6 +25,15 @@ spec:
             - name: {{ $nm }}-clusters
               configMap:
                 name: {{ $nm }}-clusters
+          {{- with .Values.nodeSelector }}
+          nodeSelector: {{  toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.affinity }}
+          affinity: {{  toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.tolerations }}
+          tolerations: {{  toYaml . | nindent 12 }}
+          {{- end }}
           containers:
             - name: setup
               image: {{ $image | quote }}

diff --git a/templates/deployment.yaml b/templates/deployment.yaml
@@ -27,6 +27,15 @@ spec:
             secretName: {{ include "cmak.name" . }}-ssl
       {{- end -}}
       {{- end }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector: {{  toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity: {{  toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations: {{  toYaml . | nindent 8 }}
+    {{- end }}
       containers:
         - name: zk
           image: {{ printf "%s/zookeeper:%s" .Values.imageRegistry .Values.zk.version | quote }}

diff --git a/templates/job-setup.yaml b/templates/job-setup.yaml
@@ -19,6 +19,15 @@ spec:
         - name: {{ $nm }}-clusters
           configMap:
             name: {{ $nm }}-clusters
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{  toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{  toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{ toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: setup
           image: {{ $image | quote }}

diff --git a/test/linter/test.sh b/test/linter/test.sh
@@ -4,3 +4,25 @@ helm lint . --strict
 helm lint . --strict -f test/linter/values-lint.yaml
 helm lint . --strict -f test/linter/values-ssl.yaml
 helm dep up test/linter/subchart && helm lint test/linter/subchart --strict
+
+helm lint . --strict --set 'tolerations.sample=1'
+if [ $? -eq 0 ]; then
+  exit 1
+fi
+
+helm lint . --strict --set 'tolerations.sample={"qwe", "asd"}'
+if [ $? -eq 0 ]; then
+  exit 1
+fi
+
+helm lint . --strict --set 'nodeSelector={1,2,3}'
+if [ $? -eq 0 ]; then
+  exit 1
+fi
+
+helm lint . --strict --set 'affinity={1,2,3}'
+if [ $? -eq 0 ]; then
+  exit 1
+fi
+
+
diff --git a/test/unit/deploy-placement.yaml b/test/unit/deploy-placement.yaml
@@ -0,0 +1,49 @@
+suite: deployment placement
+templates:
+  - deployment.yaml
+tests:
+  - it: should not populate placement by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.affinity
+      - isNull:
+          path: spec.template.spec.tolerations
+  - it: should populate nodeSelector
+    set:
+      nodeSelector:
+        myLbl: myVal
+    asserts:
+      - isNotNull:
+          path: spec.template.spec.nodeSelector
+      - isNotEmpty:
+          path: spec.template.spec.nodeSelector.myLbl
+      - equal:
+          path: spec.template.spec.nodeSelector.myLbl
+          value: myVal
+  - it: should populate affinity
+    set:
+      affinity:
+        someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.template.spec.affinity
+      - equal:
+          path: spec.template.spec.affinity.someProp
+          value: someVal
+  - it: should populate tolerations
+    set:
+      tolerations:
+        - name: val
+          someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.template.spec.tolerations
+      - contains:
+          path: spec.template.spec.tolerations
+          content:
+            name: val
+            someProp: someVal
+          count: 1
+          any: false
diff --git a/test/unit/reconcile-placement.yaml b/test/unit/reconcile-placement.yaml
@@ -0,0 +1,49 @@
+suite: reconcile cronjob placement
+templates:
+  - cronjob-reconcile.yaml
+tests:
+  - it: should not populate placement by default
+    asserts:
+      - isNull:
+          path: spec.jobTemplate.spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.jobTemplate.spec.template.spec.affinity
+      - isNull:
+          path: spec.jobTemplate.spec.template.spec.tolerations
+  - it: should populate nodeSelector
+    set:
+      nodeSelector:
+        myLbl: myVal
+    asserts:
+      - isNotNull:
+          path: spec.jobTemplate.spec.template.spec.nodeSelector
+      - isNotEmpty:
+          path: spec.jobTemplate.spec.template.spec.nodeSelector.myLbl
+      - equal:
+          path: spec.jobTemplate.spec.template.spec.nodeSelector.myLbl
+          value: myVal
+  - it: should populate affinity
+    set:
+      affinity:
+        someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.jobTemplate.spec.template.spec.affinity
+      - equal:
+          path: spec.jobTemplate.spec.template.spec.affinity.someProp
+          value: someVal
+  - it: should populate tolerations
+    set:
+      tolerations:
+        - name: val
+          someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.jobTemplate.spec.template.spec.tolerations
+      - contains:
+          path: spec.jobTemplate.spec.template.spec.tolerations
+          content:
+            name: val
+            someProp: someVal
+          count: 1
+          any: false
diff --git a/test/unit/setup-placement.yaml b/test/unit/setup-placement.yaml
@@ -0,0 +1,49 @@
+suite: setup job placement
+templates:
+  - job-setup.yaml
+tests:
+  - it: should not populate placement by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.affinity
+      - isNull:
+          path: spec.template.spec.tolerations
+  - it: should populate nodeSelector
+    set:
+      nodeSelector:
+        myLbl: myVal
+    asserts:
+      - isNotNull:
+          path: spec.template.spec.nodeSelector
+      - isNotEmpty:
+          path: spec.template.spec.nodeSelector.myLbl
+      - equal:
+          path: spec.template.spec.nodeSelector.myLbl
+          value: myVal
+  - it: should populate affinity
+    set:
+      affinity:
+        someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.template.spec.affinity
+      - equal:
+          path: spec.template.spec.affinity.someProp
+          value: someVal
+  - it: should populate tolerations
+    set:
+      tolerations:
+        - name: val
+          someProp: someVal
+    asserts:
+      - isNotEmpty:
+          path: spec.template.spec.tolerations
+      - contains:
+          path: spec.template.spec.tolerations
+          content:
+            name: val
+            someProp: someVal
+          count: 1
+          any: false
diff --git a/values.schema.json b/values.schema.json
@@ -99,11 +99,22 @@
       }
     }
   },
-
   "type": "object", "required": ["cmak", "reconcile", "ui", "zk"],
-
   "properties": {
     "imageRegistry": { "type": "string", "default": "docker.io", "title": "docker registry for all images of the chart" },
+    "affinity": {
+      "type": "object", "default": {}, "title": "affinity",
+      "description": "See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity"
+    },
+    "nodeSelector": {
+      "type": "object", "default": {}, "title": "node selector",
+      "description": "See https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/"
+    },
+    "tolerations": {
+      "type": "array", "default": [], "title": "tolerations",
+      "description": "See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/",
+      "items": {"type": "object"}
+    },
     "reconcile": {
       "type": "object", "additionalProperties": false,
       "required": ["schedule"], "title": "reconciliation job config",
@@ -132,7 +143,7 @@
         "clusters": {
           "type": "array", "title": "list of configured clusters",
           "items": { "$ref": "#/definitions/clusterConfig" }
-        }        
+        }
       }
     },
     "ui": {

diff --git a/values.yaml b/values.yaml
@@ -49,7 +49,7 @@ reconcile:
   successfulJobsHistoryLimit: null
   failedJobsHistoryLimit: null
   # reconcile resource configuration
-  # resources: 
+  # resources:
   #   limits:
   #     cpu: 100m
   #     memory: 128Mi
@@ -124,3 +124,10 @@ ingress: null
   #
   # # optional ingress labels
   # labels: {}
+
+affinity: {}
+
+nodeSelector: {}
+
+tolerations: []
+