fix(hatchery): k8s hatchery uses service account instead of kubeconfig (

#4225)

* feat(hatchery:k8s): use sa/rbac instead of embedded kubeconfig

wasn't supposed to be added

..

* revert some changes for alternate use cases, update readme

.dockerignore

@@ -0,0 +1,6 @@
+**/dist/*
+tests/
+images/
+tools/
+docs/
+contrib/

Dockerfile-dev

@@ -0,0 +1,24 @@
+FROM golang:1.12-stretch as BUILD
+
+WORKDIR /go/src/github.com/ovh/cds
+
+RUN apt-get update \
+    && apt-get install -y libsecret-1-dev
+# separating for quicker build times since its for dev
+
+COPY . .
+
+RUN OS=linux ARCH=amd64 make build \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM debian:jessie
+
+USER nobody
+WORKDIR /app
+
+COPY --from=BUILD --chown=nobody:nogroup /go/src/github.com/ovh/cds/engine/dist/* ./
+COPY --from=BUILD --chown=nobody:nogroup /go/src/github.com/ovh/cds/engine/worker/dist/* ./
+COPY --from=BUILD --chown=nobody:nogroup /go/src/github.com/ovh/cds/cli/cdsctl/dist/* ./
+COPY --chown=nobody:nogroup engine/sql sql
+
+CMD ["/app/cds-engine-linux-amd64"]

contrib/helm/cds/README.md

@@ -9,15 +9,14 @@ Documentation is available at https://ovh.github.io/cds/
 ```console
 cd contrib/helm/cds
 # To let CDS spawn workers on your kubernetes cluster you need to copy your kubeconfig.yaml in the current directory
-cp yourPathToKubeconfig.yaml kubeconfig.yaml
 helm dependency update
 helm install .
 ```
 
-
 ## FUTURE
 
 When CDS helm chart will be released you'll be able to install with
+
 ```console
 $ helm install stable/cds
 ```
@@ -36,20 +35,18 @@ It starts a PostgreSQL server, a Redis server and an Elasticsearch server using
 
 ## Prerequisites
 
-- Kubernetes 1.4+ with Beta APIs enabled
-- PV provisioner support in the underlying infrastructure
-- Kubernetes config file (`kubeconfig.yaml`) located at this path. (For minikube it's often located `~/.kube/config`)
++ Kubernetes 1.4+ with Beta APIs enabled
++ PV provisioner support in the underlying infrastructure
++ RBAC enabled or .rbac.create set to false
 
 ## Installing the Chart
 
 To install the chart with the release name `my-release`:
 
 ```console
 # Inside of cds/contrib/helm/cds
-# To let CDS spawn workers on your kubernetes cluster you need to copy your kubeconfig.yaml in the current directory
-cp yourPathToKubeconfig.yaml kubeconfig.yaml
-helm dependency update
-helm install --name my-cds . 
+$ helm dependency update
+$ helm install --name my-cds .
 ```
 
 The command deploys CDS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
@@ -153,7 +150,6 @@ And check in the log of your api server to get registration URL :
 
   export CDS_API_POD_NAME=$(kubectl get pods --namespace default -l "app=my-cds-cds-api" -o jsonpath="{.items[0].metadata.name}")
   kubectl logs -f --namespace default $CDS_API_POD_NAME | grep 'account/verify'
-
 ```
 
 + Create the first CDS User
@@ -187,8 +183,8 @@ $ chmod +x cdsctl
 *please note that the version linux/amd64, darwin/amd64 and windows/amd64 use libsecret / keychain to store the CDS Password.
 If you don't want to use the keychain, you can select the version i386*
 
-
 + Login with cdsctl
+
 ```console
 $ ./cdsctl login --api-url http://$SERVICE_IP/cdsapi -u yourusername
 CDS API URL: http://$SERVICE_IP/cdsapi
@@ -282,13 +278,9 @@ $ helm install --name my-release -f values.yaml .
 > **Tip**: You can use the default [values.yaml](values.yaml)
 
 + If you use a Kubernetes provider without LoadBalancer ability you just have to set your `ui.serviceType` to `ClusterIP` and set `ingress.enabled` to `true` with the right `ingress.hostname` and `ingress.port` (for example: `helm install --kubeconfig kubeconfig.yml --name my-release -f values.yaml --set ui.serviceType=ClusterIP --set ingress.enabled=true --set ingress.hostname=cds.MY_NODES_URL --set ingress.port=32080 .`).
-
 + If you use a minikube you have to set `ui.serviceType` to `ClusterIP`.
-
 + If you use a Kubernetes as GKE, EKS or if your cloud provider provide you an available LoadBalancer you just have to set `ui.serviceType` to `LoadBalancer`.
 
-+ Your `kubeconfig.yaml` must be located in this directory.
-
 ## Image
 
 The `image` parameter allows specifying which image will be pulled for the chart.
@@ -303,6 +295,6 @@ By default, cds api artifact directory is created as default PersistentVolumeCla
 1. Create the PersistentVolumeClaim
 1. Install the chart
 
-```bash
+```console
 $ helm install --name my-release --set cds.existingClaim=PVC_NAME .
 ```

contrib/helm/cds/templates/_helpers.tpl

@@ -38,3 +38,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- define "cds.elasticsearch.fullname" -}}
 {{- printf "%s-%s" .Release.Name "elasticsearch" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "cds.serviceAccount.name" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "cds.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

contrib/helm/cds/templates/rbac.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ template "cds.fullname" . }}
+  labels:
+    app: {{ template "cds.fullname" . }}-api
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["list", "create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ template "cds.fullname" . }}
+  labels:
+    app: {{ template "cds.fullname" . }}-api
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "cds.fullname" . }}
+subjects:
+  - name: {{ template "cds.serviceAccount.name" . }}
+    namespace: {{ .Release.Namespace | quote }}
+    kind: ServiceAccount
+{{- end -}}

contrib/helm/cds/templates/secrets.yaml

@@ -20,4 +20,3 @@ data:
   cds-api_secrets_key: {{ randAlphaNum 32 | b64enc | quote }}
   {{ end }}
   cds_config_file: {{ .Files.Get "config.toml" | b64enc | b64enc }}
-  cds-k8s_config_file: {{ .Files.Get "kubeconfig.yaml" | b64enc | b64enc }}

contrib/helm/cds/templates/serviceaccount.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+{{- if .Values.image.imagePullSecrets }}
+imagePullSecrets: {{ toYaml .Values.image.imagePullSecrets | nindent 2 }}
+{{- end }}
+metadata:
+  name: {{ template "cds.serviceAccount.name" . }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    app: {{ template "cds.fullname" . }}-api
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+  {{- end -}}

contrib/helm/cds/values.yaml

@@ -21,6 +21,16 @@ image:
 cdsUsername: cds
 logLevel: info
 
+rbac:
+  create: true
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
 ## Default: random 64 character string
 #apiAuthSharedinfratoken: changeitchangeitchangeitchangeitchangeitchangeitchangeitchangeit
 

engine/hatchery/kubernetes/kubernetes.go

@@ -10,14 +10,15 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/client-go/tools/clientcmd"
+
 	"github.com/gorilla/mux"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	"github.com/ovh/cds/engine/api"
@@ -59,47 +60,62 @@ func (h *HatcheryKubernetes) ApplyConfiguration(cfg interface{}) error {
 	}
 
 	var errCl error
-	var clientset *kubernetes.Clientset
+	var clientSet *kubernetes.Clientset
 	k8sTimeout := time.Second * 10
-	if h.Config.KubernetesConfigFile != "" {
-		cfg, err := clientcmd.BuildConfigFromFlags(h.Config.KubernetesMasterURL, h.Config.KubernetesConfigFile)
-		if err != nil {
-			return sdk.WrapError(err, "Cannot build config from flags")
-		}
-		cfg.Timeout = k8sTimeout
 
-		clientset, errCl = kubernetes.NewForConfig(cfg)
-		if errCl != nil {
-			return sdk.WrapError(errCl, "Cannot create client with newForConfig")
+	if h.Config.KubernetesMasterURL != "" {
+		if h.Config.KubernetesConfigFile != "" {
+			cfg, err := clientcmd.BuildConfigFromFlags(h.Config.KubernetesMasterURL, h.Config.KubernetesConfigFile)
+			if err != nil {
+				return sdk.WrapError(err, "Cannot build config from flags")
+			}
+			cfg.Timeout = k8sTimeout
+
+			clientSet, errCl = kubernetes.NewForConfig(cfg)
+			if errCl != nil {
+				return sdk.WrapError(errCl, "Cannot create client with newForConfig")
+			}
+		} else {
+			configK8s, err := clientcmd.BuildConfigFromKubeconfigGetter(h.Config.KubernetesMasterURL, h.getStartingConfig)
+			if err != nil {
+				return sdk.WrapError(err, "Cannot build config from config getter")
+			}
+			configK8s.Timeout = k8sTimeout
+
+			if h.Config.KubernetesCertAuthData != "" {
+				configK8s.TLSClientConfig = rest.TLSClientConfig{
+					CAData:   []byte(h.Config.KubernetesCertAuthData),
+					CertData: []byte(h.Config.KubernetesClientCertData),
+					KeyData:  []byte(h.Config.KubernetesClientKeyData),
+				}
+			}
+
+			// creates the clientset
+			clientSet, errCl = kubernetes.NewForConfig(configK8s)
+			if errCl != nil {
+				return sdk.WrapError(errCl, "Cannot create new config")
+			}
 		}
 	} else {
-		configK8s, err := clientcmd.BuildConfigFromKubeconfigGetter(h.Config.KubernetesMasterURL, h.getStartingConfig)
+		config, err := rest.InClusterConfig()
 		if err != nil {
-			return sdk.WrapError(err, "Cannot build config from config getter")
-		}
-		configK8s.Timeout = k8sTimeout
-
-		if h.Config.KubernetesCertAuthData != "" {
-			configK8s.TLSClientConfig = rest.TLSClientConfig{
-				CAData:   []byte(h.Config.KubernetesCertAuthData),
-				CertData: []byte(h.Config.KubernetesClientCertData),
-				KeyData:  []byte(h.Config.KubernetesClientKeyData),
-			}
+			return sdk.WrapError(err, "Unable to configure k8s InClusterConfig")
 		}
 
-		// creates the clientset
-		clientset, errCl = kubernetes.NewForConfig(configK8s)
+		clientSet, errCl = kubernetes.NewForConfig(config)
 		if errCl != nil {
-			return sdk.WrapError(errCl, "Cannot create new config")
+			return sdk.WrapError(errCl, "Unable to configure k8s client with InClusterConfig")
 		}
+
 	}
-	h.k8sClient = clientset
+
+	h.k8sClient = clientSet
 
 	if h.Config.Namespace != apiv1.NamespaceDefault {
-		if _, err := clientset.CoreV1().Namespaces().Get(h.Config.Namespace, metav1.GetOptions{}); err != nil {
+		if _, err := clientSet.CoreV1().Namespaces().Get(h.Config.Namespace, metav1.GetOptions{}); err != nil {
 			ns := apiv1.Namespace{}
 			ns.SetName(h.Config.Namespace)
-			if _, errC := clientset.CoreV1().Namespaces().Create(&ns); errC != nil {
+			if _, errC := clientSet.CoreV1().Namespaces().Create(&ns); errC != nil {
 				return sdk.WrapError(errC, "Cannot create namespace %s in kubernetes", h.Config.Namespace)
 			}
 		}
@@ -173,10 +189,6 @@ func (h *HatcheryKubernetes) CheckConfiguration(cfg interface{}) error {
 		return fmt.Errorf("please enter a valid kubernetes namespace")
 	}
 
-	if hconfig.KubernetesMasterURL == "" && hconfig.KubernetesConfigFile == "" {
-		return fmt.Errorf("please enter a valid kubernetes master URL or provide a kubernetes config file")
-	}
-
 	return nil
 }