Merge pull request #7 from kfox1111/container-lvmd

Add support for running lvmd as a container - WIP

charts/topolvm/Chart.yaml

@@ -6,4 +6,4 @@ version: 1.1.1
 home: https://github.com/topolvm/topolvm
 sources:
   - https://github.com/topolvm/helm/tree/master/charts/topolvm
-  - https://github.com/topolvm/topolvm
+  - https://github.com/topolvm/topolvm

charts/topolvm/README.md

@@ -38,3 +38,9 @@ Alternatively a YAML file that specifies the values for the parameters can be pr
 helm upgrade -i topolvm -f values.yaml charts/topolvm
 ```
 
+---
+
+Dont forget to:
+kubectl label namespace kube-system topolvm.cybozu.com/webhook=ignore
+install the kube-scheduler plugin as described in the "Configure kube-scheduler" section of /deploy/README.md
+Config is automatically copied to the masters at /etc/topolvm/scheduler when deployed as a daemonset and kubeScheduler.managed=true

charts/topolvm/templates/kube-scheduler/clusterrole.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.kubeScheduler.managed }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Namespace }}:kubeScheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+rules:
+  - apiGroups: ["policy"]
+    resources: ["podsecuritypolicies"]
+    verbs: ["use"]
+    resourceNames: ["{{ template "topolvm.fullname" . }}-kubescheduler"]
+{{- end }}

charts/topolvm/templates/kube-scheduler/clusterrolebinding.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.kubeScheduler.managed }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Namespace }}:kubescheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "topolvm.fullname" . }}-lvmd
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Namespace }}:kubescheduler
+{{- end }}

charts/topolvm/templates/kube-scheduler/configmap.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.kubeScheduler.managed }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "topolvm.fullname" . }}-kubescheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+data:
+  scheduler-config.yaml: |
+{{ toYaml .Values.kubeScheduler.config | indent 4 }}
+  scheduler-policy.json: |
+{{ toJson .Values.kubeScheduler.policy | indent 4 }}
+{{- end }}

charts/topolvm/templates/kube-scheduler/daemonset.yaml

@@ -0,0 +1,71 @@
+{{ if and (eq (lower .Values.scheduler.type) "daemonset") .Values.kubeScheduler.managed }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ template "topolvm.fullname" . }}-kubescheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+spec:
+  updateStrategy:
+{{ toYaml .Values.scheduler.updateStrategy | indent 4 }}
+  minReadySeconds: {{ .Values.scheduler.minReadySeconds }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-kubescheduler
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-kubescheduler
+    spec:
+      serviceAccountName: {{ template "topolvm.fullname" . }}-kubescheduler
+      initContainers:
+        - name: topolvm-kubescheduler-init
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          securityContext:
+            privileged: true
+          command:
+            - /bin/sh
+            - -ec
+            - |
+              cat /config/scheduler-config.yaml > /etc/topolvm/scheduler/.scheduler-config.yaml
+              cat /config/scheduler-policy.json > /etc/topolvm/scheduler/.scheduler-policy.json
+              mv /etc/topolvm/scheduler/.scheduler-config.yaml /etc/topolvm/scheduler/scheduler-config.yaml
+              mv /etc/topolvm/scheduler/.scheduler-policy.json /etc/topolvm/scheduler/scheduler-policy.json
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /etc/topolvm/scheduler
+              name: hostconfig
+          resources:
+{{ toYaml .Values.kubeScheduler.resources | indent 12 }}
+      containers:
+        - name: topolvm-kubescheduler
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          command:
+            - /bin/sh
+            - -c
+            - while true; do sleep 1000; done
+          resources:
+{{ toYaml .Values.kubeScheduler.resources | indent 12 }}
+      hostNetwork: true
+      {{- with .Values.scheduler.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.scheduler.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- if .Values.scheduler.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.scheduler.nodeSelector | indent 8 }}
+      {{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ template "topolvm.fullname" . }}-kubescheduler
+        - name: hostconfig
+          hostPath:
+            path: /etc/topolvm/scheduler
+            type: DirectoryOrCreate
+{{ end }}

charts/topolvm/templates/kube-scheduler/psp.yaml

@@ -0,0 +1,27 @@
+{{- if and .Values.podSecurityPolicy.create .Values.kubeScheduler.managed }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ template "topolvm.fullname" . }}-kubescheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  volumes:
+    - 'configMap'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: "/etc/topolvm/scheduler"
+      readOnly: false
+  hostNetwork: true
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: true
+{{- end }}

charts/topolvm/templates/kube-scheduler/serviceaccount.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.kubeScheduler.managed }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "topolvm.fullname" . }}-kubescheduler
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+{{- end }}

charts/topolvm/templates/lvmd/clusterrole.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.lvmd.managed }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Namespace }}:lvmd
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+rules:
+  - apiGroups: ["policy"]
+    resources: ["podsecuritypolicies"]
+    verbs: ["use"]
+    resourceNames: ["{{ template "topolvm.fullname" . }}-lvmd"]
+{{- end }}

charts/topolvm/templates/lvmd/clusterrolebinding.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.lvmd.managed }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Namespace }}:lvmd
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "topolvm.fullname" . }}-lvmd
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Namespace }}:lvmd
+{{- end }}

charts/topolvm/templates/lvmd/configmap.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.lvmd.managed }}
+{{ $global := . }}
+{{- $lvmds := concat ( list .Values.lvmd ) .Values.lvmd.additionalConfigs }}
+{{- range $lvmdidx, $lvmd := $lvmds }}
+{{- with $global }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }}
+  labels:
+    idx: {{ $lvmdidx | quote}}
+  {{- include "topolvm.labels" . | nindent 4 }}
+data:
+  lvmd.yaml: |
+    socket-name: /run/topolvm/lvmd.sock
+    device-classes:
+{{ toYaml $lvmd.deviceClasses | indent 6 }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/topolvm/templates/lvmd/daemonset.yaml

@@ -0,0 +1,64 @@
+{{- if .Values.lvmd.managed }}
+{{ $global := . }}
+{{- $lvmds := concat ( list .Values.lvmd ) .Values.lvmd.additionalConfigs }}
+{{- range $lvmdidx, $lvmd := $lvmds }}
+{{- with $global }}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }}
+  labels:
+    idx: {{ $lvmdidx | quote}}
+  {{- include "topolvm.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      idx: {{ $lvmdidx | quote}}
+      app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }}
+  template:
+    metadata:
+      labels:
+        idx: {{ $lvmdidx | quote}}
+        app.kubernetes.io/name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }}
+      annotations:
+        {{ if and .Values.node.metrics.enabled .Values.node.metrics.annotations }}
+        {{ toYaml .Values.node.metrics.annotations | indent 8 }}
+        {{ end }}
+    spec:
+      serviceAccountName: {{ template "topolvm.fullname" . }}-lvmd
+      hostPID: true
+      containers:
+        - name: lvmd
+          image: "{{ .Values.lvmd.image.repository | default .Values.image.repository }}:{{ .Values.lvmd.image.tag | default .Values.image.tag }}"
+          securityContext:
+            privileged: true
+          command:
+            - /lvmd
+            - --container
+          resources:
+            {{- toYaml .Values.lvmd.resources | nindent 12 }}
+          volumeMounts:
+            - name: lvmd-socket-dir
+              mountPath: /run/topolvm
+            - name: config
+              mountPath: /etc/topolvm
+      volumes:
+        - name: config
+          configMap:
+            name: {{ template "topolvm.fullname" . }}-lvmd{{ $lvmdidx }}
+        - name: lvmd-socket-dir
+          hostPath:
+            path: /run/topolvm
+            type: DirectoryOrCreate
+    {{- if $lvmd.tolerations }}
+      tolerations:
+{{ toYaml $lvmd.tolerations | indent 8 }}
+    {{- end }}
+    {{- if $lvmd.nodeSelector }}
+      nodeSelector:
+{{ toYaml $lvmd.nodeSelector | indent 8 }}
+    {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/topolvm/templates/lvmd/psp.yaml

@@ -0,0 +1,27 @@
+{{- if and .Values.podSecurityPolicy.create .Values.lvmd.managed }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ template "topolvm.fullname" . }}-lvmd
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  volumes:
+    - 'configMap'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: "/run/topolvm"
+      readOnly: false
+  hostNetwork: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: true
+{{- end }}

charts/topolvm/templates/lvmd/serviceaccount.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.lvmd.managed }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "topolvm.fullname" . }}-lvmd
+  labels:
+  {{- include "topolvm.labels" . | nindent 4 }}
+{{- end }}