From b00d1dbd5b7f6a48497441daa3c0d77fb893d7a3 Mon Sep 17 00:00:00 2001 From: Kevin Fox Date: Tue, 29 Sep 2020 15:38:15 -0700 Subject: [PATCH] Add support for running lvmd as a container --- charts/topolvm/Chart.yaml | 4 +- charts/topolvm/README.md | 6 ++ .../templates/kube-scheduler/clusterrole.yaml | 13 ++++ .../kube-scheduler/clusterrolebinding.yaml | 16 +++++ .../templates/kube-scheduler/configmap.yaml | 13 ++++ .../templates/kube-scheduler/daemonset.yaml | 71 ++++++++++++++++++ .../topolvm/templates/kube-scheduler/psp.yaml | 27 +++++++ .../kube-scheduler/serviceaccount.yaml | 8 +++ .../topolvm/templates/lvmd/clusterrole.yaml | 13 ++++ .../templates/lvmd/clusterrolebinding.yaml | 16 +++++ charts/topolvm/templates/lvmd/configmap.yaml | 21 ++++++ charts/topolvm/templates/lvmd/daemonset.yaml | 64 +++++++++++++++++ charts/topolvm/templates/lvmd/psp.yaml | 27 +++++++ .../templates/lvmd/serviceaccount.yaml | 8 +++ charts/topolvm/values.yaml | 72 +++++++++++++++++++ 15 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 charts/topolvm/templates/kube-scheduler/clusterrole.yaml create mode 100644 charts/topolvm/templates/kube-scheduler/clusterrolebinding.yaml create mode 100644 charts/topolvm/templates/kube-scheduler/configmap.yaml create mode 100644 charts/topolvm/templates/kube-scheduler/daemonset.yaml create mode 100644 charts/topolvm/templates/kube-scheduler/psp.yaml create mode 100644 charts/topolvm/templates/kube-scheduler/serviceaccount.yaml create mode 100644 charts/topolvm/templates/lvmd/clusterrole.yaml create mode 100644 charts/topolvm/templates/lvmd/clusterrolebinding.yaml create mode 100644 charts/topolvm/templates/lvmd/configmap.yaml create mode 100644 charts/topolvm/templates/lvmd/daemonset.yaml create mode 100644 charts/topolvm/templates/lvmd/psp.yaml create mode 100644 charts/topolvm/templates/lvmd/serviceaccount.yaml diff --git a/charts/topolvm/Chart.yaml b/charts/topolvm/Chart.yaml index 2be03fc..ebbd008 100755 --- a/charts/topolvm/Chart.yaml +++ b/charts/topolvm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v1 appVersion: 0.6.0 description: Topolvm name: topolvm -version: 1.0.1 +version: 1.0.2 home: https://github.com/topolvm/topolvm sources: - https://github.com/topolvm/helm/tree/master/charts/topolvm - - https://github.com/topolvm/topolvm \ No newline at end of file + - https://github.com/topolvm/topolvm diff --git a/charts/topolvm/README.md b/charts/topolvm/README.md index eda3c67..6ee983c 100644 --- a/charts/topolvm/README.md +++ b/charts/topolvm/README.md @@ -37,3 +37,9 @@ Alternatively a YAML file that specifies the values for the parameters can be pr helm upgrade -i topolvm -f values.yaml charts/topolvm ``` +--- + +Dont forget to: +kubectl label namespace kube-system topolvm.cybozu.com/webhook=ignore +install the kube-scheduler plugin as described in the "Configure kube-scheduler" section of /deploy/README.md +Config is automatically copied to the masters at /etc/topolvm/scheduler when deployed as a daemonset and kubeScheduler.managed=true diff --git a/charts/topolvm/templates/kube-scheduler/clusterrole.yaml b/charts/topolvm/templates/kube-scheduler/clusterrole.yaml new file mode 100644 index 0000000..70866a6 --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/clusterrole.yaml @@ -0,0 +1,13 @@ +{{- if .Values.kubeScheduler.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}:kubeScheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +rules: + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: ["{{ template "topolvm.fullname" . }}-kubescheduler"] +{{- end }} diff --git a/charts/topolvm/templates/kube-scheduler/clusterrolebinding.yaml b/charts/topolvm/templates/kube-scheduler/clusterrolebinding.yaml new file mode 100644 index 0000000..6ab723a --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeScheduler.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}:kubescheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ template "topolvm.fullname" . }}-lvmd + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}:kubescheduler +{{- end }} diff --git a/charts/topolvm/templates/kube-scheduler/configmap.yaml b/charts/topolvm/templates/kube-scheduler/configmap.yaml new file mode 100644 index 0000000..0d1760c --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.kubeScheduler.managed }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "topolvm.fullname" . }}-kubescheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +data: + scheduler-config.yaml: | +{{ toYaml .Values.kubeScheduler.config | indent 4 }} + scheduler-policy.json: | +{{ toJson .Values.kubeScheduler.policy | indent 4 }} +{{- end }} diff --git a/charts/topolvm/templates/kube-scheduler/daemonset.yaml b/charts/topolvm/templates/kube-scheduler/daemonset.yaml new file mode 100644 index 0000000..6cd66f9 --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/daemonset.yaml @@ -0,0 +1,71 @@ +{{ if and (eq (lower .Values.scheduler.type) "daemonset") .Values.kubeScheduler.managed }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "topolvm.fullname" . }}-kubescheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +spec: + updateStrategy: +{{ toYaml .Values.scheduler.updateStrategy | indent 4 }} + minReadySeconds: {{ .Values.scheduler.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-kubescheduler + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-kubescheduler + spec: + serviceAccountName: {{ template "topolvm.fullname" . }}-kubescheduler + initContainers: + - name: topolvm-kubescheduler-init + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + privileged: true + command: + - /bin/sh + - -ec + - | + cat /config/scheduler-config.yaml > /etc/topolvm/scheduler/.scheduler-config.yaml + cat /config/scheduler-policy.json > /etc/topolvm/scheduler/.scheduler-policy.json + mv /etc/topolvm/scheduler/.scheduler-config.yaml /etc/topolvm/scheduler/scheduler-config.yaml + mv /etc/topolvm/scheduler/.scheduler-policy.json /etc/topolvm/scheduler/scheduler-policy.json + volumeMounts: + - mountPath: /config + name: config + - mountPath: /etc/topolvm/scheduler + name: hostconfig + resources: +{{ toYaml .Values.kubeScheduler.resources | indent 12 }} + containers: + - name: topolvm-kubescheduler + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - /bin/sh + - -c + - while true; do sleep 1000; done + resources: +{{ toYaml .Values.kubeScheduler.resources | indent 12 }} + hostNetwork: true + {{- with .Values.scheduler.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.scheduler.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.scheduler.nodeSelector | indent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "topolvm.fullname" . }}-kubescheduler + - name: hostconfig + hostPath: + path: /etc/topolvm/scheduler + type: DirectoryOrCreate +{{ end }} diff --git a/charts/topolvm/templates/kube-scheduler/psp.yaml b/charts/topolvm/templates/kube-scheduler/psp.yaml new file mode 100644 index 0000000..0945e36 --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/psp.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.podSecurityPolicy.create .Values.kubeScheduler.managed }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "topolvm.fullname" . }}-kubescheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +spec: + privileged: true + allowPrivilegeEscalation: true + volumes: + - 'configMap' + - 'hostPath' + allowedHostPaths: + - pathPrefix: "/etc/topolvm/scheduler" + readOnly: false + hostNetwork: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: true +{{- end }} diff --git a/charts/topolvm/templates/kube-scheduler/serviceaccount.yaml b/charts/topolvm/templates/kube-scheduler/serviceaccount.yaml new file mode 100644 index 0000000..8799031 --- /dev/null +++ b/charts/topolvm/templates/kube-scheduler/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.kubeScheduler.managed }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "topolvm.fullname" . }}-kubescheduler + labels: + {{- include "topolvm.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/topolvm/templates/lvmd/clusterrole.yaml b/charts/topolvm/templates/lvmd/clusterrole.yaml new file mode 100644 index 0000000..c9be620 --- /dev/null +++ b/charts/topolvm/templates/lvmd/clusterrole.yaml @@ -0,0 +1,13 @@ +{{- if .Values.lvmd.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}:lvmd + labels: + {{- include "topolvm.labels" . | nindent 4 }} +rules: + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: ["{{ template "topolvm.fullname" . }}-lvmd"] +{{- end }} diff --git a/charts/topolvm/templates/lvmd/clusterrolebinding.yaml b/charts/topolvm/templates/lvmd/clusterrolebinding.yaml new file mode 100644 index 0000000..d012b2d --- /dev/null +++ b/charts/topolvm/templates/lvmd/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.lvmd.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}:lvmd + labels: + {{- include "topolvm.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ template "topolvm.fullname" . }}-lvmd + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}:lvmd +{{- end }} diff --git a/charts/topolvm/templates/lvmd/configmap.yaml b/charts/topolvm/templates/lvmd/configmap.yaml new file mode 100644 index 0000000..2b55728 --- /dev/null +++ b/charts/topolvm/templates/lvmd/configmap.yaml @@ -0,0 +1,21 @@ +{{- if .Values.lvmd.managed }} +{{ $global := . }} +{{- $lvmds := concat ( list .Values.lvmd ) .Values.lvmd.additionalConfigs }} +{{- range $lvmdidx, $lvmd := $lvmds }} +{{- with $global }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }} + labels: + idx: {{ $lvmdidx | quote}} + {{- include "topolvm.labels" . | nindent 4 }} +data: + lvmd.yaml: | + socket-name: /run/topolvm/lvmd.sock + device-classes: +{{ toYaml $lvmd.deviceClasses | indent 6 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/topolvm/templates/lvmd/daemonset.yaml b/charts/topolvm/templates/lvmd/daemonset.yaml new file mode 100644 index 0000000..3c4e73f --- /dev/null +++ b/charts/topolvm/templates/lvmd/daemonset.yaml @@ -0,0 +1,64 @@ +{{- if .Values.lvmd.managed }} +{{ $global := . }} +{{- $lvmds := concat ( list .Values.lvmd ) .Values.lvmd.additionalConfigs }} +{{- range $lvmdidx, $lvmd := $lvmds }} +{{- with $global }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }} + labels: + idx: {{ $lvmdidx | quote}} + {{- include "topolvm.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + idx: {{ $lvmdidx | quote}} + app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }} + template: + metadata: + labels: + idx: {{ $lvmdidx | quote}} + app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }} + annotations: + {{ if and .Values.node.metrics.enabled .Values.node.metrics.annotations }} + {{ toYaml .Values.node.metrics.annotations | indent 8 }} + {{ end }} + spec: + serviceAccountName: {{ template "topolvm.fullname" . }}-lvmd + hostPID: true + containers: + - name: lvmd + image: "{{ .Values.lvmd.image.repository | default .Values.image.repository }}:{{ .Values.lvmd.image.tag | default .Values.image.tag }}" + securityContext: + privileged: true + command: + - /lvmd + - --container + resources: + {{- toYaml .Values.lvmd.resources | nindent 12 }} + volumeMounts: + - name: lvmd-socket-dir + mountPath: /run/topolvm + - name: config + mountPath: /etc/topolvm + volumes: + - name: config + configMap: + name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }} + - name: lvmd-socket-dir + hostPath: + path: /run/topolvm + type: DirectoryOrCreate + {{- if $lvmd.tolerations }} + tolerations: +{{ toYaml $lvmd.tolerations | indent 8 }} + {{- end }} + {{- if $lvmd.nodeSelector }} + nodeSelector: +{{ toYaml $lvmd.nodeSelector | indent 8 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/topolvm/templates/lvmd/psp.yaml b/charts/topolvm/templates/lvmd/psp.yaml new file mode 100644 index 0000000..edcbfe0 --- /dev/null +++ b/charts/topolvm/templates/lvmd/psp.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.podSecurityPolicy.create .Values.lvmd.managed }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "topolvm.fullname" . }}-lvmd + labels: + {{- include "topolvm.labels" . | nindent 4 }} +spec: + privileged: true + allowPrivilegeEscalation: true + volumes: + - 'configMap' + - 'hostPath' + allowedHostPaths: + - pathPrefix: "/run/topolvm" + readOnly: false + hostNetwork: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: true +{{- end }} diff --git a/charts/topolvm/templates/lvmd/serviceaccount.yaml b/charts/topolvm/templates/lvmd/serviceaccount.yaml new file mode 100644 index 0000000..118dd4a --- /dev/null +++ b/charts/topolvm/templates/lvmd/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.lvmd.managed }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "topolvm.fullname" . }}-lvmd + labels: + {{- include "topolvm.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/topolvm/values.yaml b/charts/topolvm/values.yaml index b343578..8edf5cc 100755 --- a/charts/topolvm/values.yaml +++ b/charts/topolvm/values.yaml @@ -77,7 +77,79 @@ scheduler: host: localhost port: 9251 +kubeScheduler: + managed: false + config: + apiVersion: kubescheduler.config.k8s.io/v1alpha1 + kind: KubeSchedulerConfiguration + schedulerName: default-scheduler + algorithmSource: + policy: + file: + path: /etc/topolvm/scheduler/scheduler-policy.cfg + leaderElection: + leaderElect: true + clientConnection: + kubeconfig: /etc/kubernetes/scheduler.conf + policy: + kind: Policy + apiVersion: v1 + extenders: + - urlPrefix: http://127.0.0.1:9251 + filterVerb: predicate + prioritizeVerb: prioritize + nodeCacheCapable: false + weight: 1 + managedResources: + - name: topolvm.cybozu.com/capacity + ignoredByScheduler: true + resources: {} +# requests: +# memory: 50Mi +# cpu: 10m +# limits: +# memory: 50Mi +# cpu: 10m + +# lvmd service +lvmd: + managed: false + deviceClasses: [] +# - name: ssd +# volume-group: myvg1 +# default: true +# spare-gb: 10 + image: + repository: + tag: + resources: {} +# requests: +# memory: 100Mi +# cpu: 100m +# limits: +# memory: 500Mi +# cpu: 500m + ## Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + ## + tolerations: [] + + ## Node labels for node pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + nodeSelector: {} + ## Define additional LVM Daemon configs if you have additional types of nodes. + ## Please ensure nodeSelectors are non overlapping. + ## + additionalConfigs: [] +# - tolerations: [] +# nodeSelector: {} +# device-classes: +# - name: ssd +# volume-group: myvg2 +# default: true +# spare-gb: 10 # CSI node service node: