From 5733852f11cd89ea40e1c3bf92f1ca3aa49a4269 Mon Sep 17 00:00:00 2001 From: dmakeroam Date: Wed, 6 Oct 2021 16:35:16 +0700 Subject: [PATCH] feat: add nodeSelector, pod tolerations, node affinity configuration --- .github/workflows/ci.yaml | 2 +- templates/cronjob-reconcile.yaml | 9 ++++++ templates/deployment.yaml | 9 ++++++ templates/job-setup.yaml | 9 ++++++ test/linter/test.sh | 22 ++++++++++++++ test/unit/deploy-placement.yaml | 49 ++++++++++++++++++++++++++++++ test/unit/reconcile-placement.yaml | 49 ++++++++++++++++++++++++++++++ test/unit/setup-placement.yaml | 49 ++++++++++++++++++++++++++++++ values.schema.json | 17 +++++++++-- values.yaml | 9 +++++- 10 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 test/unit/deploy-placement.yaml create mode 100644 test/unit/reconcile-placement.yaml create mode 100644 test/unit/setup-placement.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a81a36b..07b94ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: - run: ./test/linter/test.sh - run: | helm plugin install https://github.com/quintush/helm-unittest - helm unittest -f test/unit/*.yaml -3 . + helm unittest -f 'test/unit/*.yaml' -3 . - uses: AbsaOSS/k3d-action@v1.5.0 with: cluster-name: "k3d" diff --git a/templates/cronjob-reconcile.yaml b/templates/cronjob-reconcile.yaml index 4391df2..9da03a7 100644 --- a/templates/cronjob-reconcile.yaml +++ b/templates/cronjob-reconcile.yaml @@ -25,6 +25,15 @@ spec: - name: {{ $nm }}-clusters configMap: name: {{ $nm }}-clusters + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{ toYaml . | nindent 12 }} + {{- end }} containers: - name: setup image: {{ $image | quote }} diff --git a/templates/deployment.yaml b/templates/deployment.yaml index aaaacb2..170552e 100644 --- a/templates/deployment.yaml +++ b/templates/deployment.yaml @@ -27,6 +27,15 @@ spec: secretName: {{ include "cmak.name" . }}-ssl {{- end -}} {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} containers: - name: zk image: {{ printf "%s/zookeeper:%s" .Values.imageRegistry .Values.zk.version | quote }} diff --git a/templates/job-setup.yaml b/templates/job-setup.yaml index a397935..c8a142d 100644 --- a/templates/job-setup.yaml +++ b/templates/job-setup.yaml @@ -19,6 +19,15 @@ spec: - name: {{ $nm }}-clusters configMap: name: {{ $nm }}-clusters + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} containers: - name: setup image: {{ $image | quote }} diff --git a/test/linter/test.sh b/test/linter/test.sh index d39e937..edaa811 100755 --- a/test/linter/test.sh +++ b/test/linter/test.sh @@ -4,3 +4,25 @@ helm lint . --strict helm lint . --strict -f test/linter/values-lint.yaml helm lint . --strict -f test/linter/values-ssl.yaml helm dep up test/linter/subchart && helm lint test/linter/subchart --strict + +helm lint . --strict --set 'tolerations.sample=1' +if [ $? -eq 0 ]; then + exit 1 +fi + +helm lint . --strict --set 'tolerations.sample={"qwe", "asd"}' +if [ $? -eq 0 ]; then + exit 1 +fi + +helm lint . --strict --set 'nodeSelector={1,2,3}' +if [ $? -eq 0 ]; then + exit 1 +fi + +helm lint . --strict --set 'affinity={1,2,3}' +if [ $? -eq 0 ]; then + exit 1 +fi + + diff --git a/test/unit/deploy-placement.yaml b/test/unit/deploy-placement.yaml new file mode 100644 index 0000000..101db4c --- /dev/null +++ b/test/unit/deploy-placement.yaml @@ -0,0 +1,49 @@ +suite: deployment placement +templates: + - deployment.yaml +tests: + - it: should not populate placement by default + asserts: + - isNull: + path: spec.template.spec.nodeSelector + - isNull: + path: spec.template.spec.affinity + - isNull: + path: spec.template.spec.tolerations + - it: should populate nodeSelector + set: + nodeSelector: + myLbl: myVal + asserts: + - isNotNull: + path: spec.template.spec.nodeSelector + - isNotEmpty: + path: spec.template.spec.nodeSelector.myLbl + - equal: + path: spec.template.spec.nodeSelector.myLbl + value: myVal + - it: should populate affinity + set: + affinity: + someProp: someVal + asserts: + - isNotEmpty: + path: spec.template.spec.affinity + - equal: + path: spec.template.spec.affinity.someProp + value: someVal + - it: should populate tolerations + set: + tolerations: + - name: val + someProp: someVal + asserts: + - isNotEmpty: + path: spec.template.spec.tolerations + - contains: + path: spec.template.spec.tolerations + content: + name: val + someProp: someVal + count: 1 + any: false diff --git a/test/unit/reconcile-placement.yaml b/test/unit/reconcile-placement.yaml new file mode 100644 index 0000000..b989380 --- /dev/null +++ b/test/unit/reconcile-placement.yaml @@ -0,0 +1,49 @@ +suite: reconcile cronjob placement +templates: + - cronjob-reconcile.yaml +tests: + - it: should not populate placement by default + asserts: + - isNull: + path: spec.jobTemplate.spec.template.spec.nodeSelector + - isNull: + path: spec.jobTemplate.spec.template.spec.affinity + - isNull: + path: spec.jobTemplate.spec.template.spec.tolerations + - it: should populate nodeSelector + set: + nodeSelector: + myLbl: myVal + asserts: + - isNotNull: + path: spec.jobTemplate.spec.template.spec.nodeSelector + - isNotEmpty: + path: spec.jobTemplate.spec.template.spec.nodeSelector.myLbl + - equal: + path: spec.jobTemplate.spec.template.spec.nodeSelector.myLbl + value: myVal + - it: should populate affinity + set: + affinity: + someProp: someVal + asserts: + - isNotEmpty: + path: spec.jobTemplate.spec.template.spec.affinity + - equal: + path: spec.jobTemplate.spec.template.spec.affinity.someProp + value: someVal + - it: should populate tolerations + set: + tolerations: + - name: val + someProp: someVal + asserts: + - isNotEmpty: + path: spec.jobTemplate.spec.template.spec.tolerations + - contains: + path: spec.jobTemplate.spec.template.spec.tolerations + content: + name: val + someProp: someVal + count: 1 + any: false diff --git a/test/unit/setup-placement.yaml b/test/unit/setup-placement.yaml new file mode 100644 index 0000000..349f8a2 --- /dev/null +++ b/test/unit/setup-placement.yaml @@ -0,0 +1,49 @@ +suite: setup job placement +templates: + - job-setup.yaml +tests: + - it: should not populate placement by default + asserts: + - isNull: + path: spec.template.spec.nodeSelector + - isNull: + path: spec.template.spec.affinity + - isNull: + path: spec.template.spec.tolerations + - it: should populate nodeSelector + set: + nodeSelector: + myLbl: myVal + asserts: + - isNotNull: + path: spec.template.spec.nodeSelector + - isNotEmpty: + path: spec.template.spec.nodeSelector.myLbl + - equal: + path: spec.template.spec.nodeSelector.myLbl + value: myVal + - it: should populate affinity + set: + affinity: + someProp: someVal + asserts: + - isNotEmpty: + path: spec.template.spec.affinity + - equal: + path: spec.template.spec.affinity.someProp + value: someVal + - it: should populate tolerations + set: + tolerations: + - name: val + someProp: someVal + asserts: + - isNotEmpty: + path: spec.template.spec.tolerations + - contains: + path: spec.template.spec.tolerations + content: + name: val + someProp: someVal + count: 1 + any: false diff --git a/values.schema.json b/values.schema.json index 8e076d6..b388a5c 100644 --- a/values.schema.json +++ b/values.schema.json @@ -99,11 +99,22 @@ } } }, - "type": "object", "required": ["cmak", "reconcile", "ui", "zk"], - "properties": { "imageRegistry": { "type": "string", "default": "docker.io", "title": "docker registry for all images of the chart" }, + "affinity": { + "type": "object", "default": {}, "title": "affinity", + "description": "See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity" + }, + "nodeSelector": { + "type": "object", "default": {}, "title": "node selector", + "description": "See https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/" + }, + "tolerations": { + "type": "array", "default": [], "title": "tolerations", + "description": "See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/", + "items": {"type": "object"} + }, "reconcile": { "type": "object", "additionalProperties": false, "required": ["schedule"], "title": "reconciliation job config", @@ -132,7 +143,7 @@ "clusters": { "type": "array", "title": "list of configured clusters", "items": { "$ref": "#/definitions/clusterConfig" } - } + } } }, "ui": { diff --git a/values.yaml b/values.yaml index 486aa3e..be5b5b8 100644 --- a/values.yaml +++ b/values.yaml @@ -49,7 +49,7 @@ reconcile: successfulJobsHistoryLimit: null failedJobsHistoryLimit: null # reconcile resource configuration - # resources: + # resources: # limits: # cpu: 100m # memory: 128Mi @@ -124,3 +124,10 @@ ingress: null # # # optional ingress labels # labels: {} + +affinity: {} + +nodeSelector: {} + +tolerations: [] +