diff --git a/Makefile b/Makefile index cd382dd..6d1dfea 100644 --- a/Makefile +++ b/Makefile @@ -26,4 +26,5 @@ deploy: apply \ --filename deploy/cluster-role.yaml \ --filename deploy/dummy.yaml \ - --filename deploy/deployment.yaml + --filename deploy/deployment-fargate.yaml \ + --filename deploy/deployment-node-exporter.yaml diff --git a/deploy/cluster-role.yaml b/deploy/cluster-role.yaml index 7ba5ed2..e3835d1 100644 --- a/deploy/cluster-role.yaml +++ b/deploy/cluster-role.yaml @@ -1,5 +1,14 @@ --- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: kube-sidecar-injector + labels: + app.kubernetes.io/name: kube-sidecar-injector + +--- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -10,3 +19,20 @@ rules: - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations"] verbs: ["create", "get", "delete", "list", "patch", "update", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-sidecar-injector + labels: + app.kubernetes.io/name: kube-sidecar-injector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-sidecar-injector +subjects: + - kind: ServiceAccount + name: kube-sidecar-injector + namespace: default diff --git a/deploy/deployment-fargate.yaml b/deploy/deployment-fargate.yaml new file mode 100644 index 0000000..3cddb76 --- /dev/null +++ b/deploy/deployment-fargate.yaml @@ -0,0 +1,75 @@ +--- + +kind: Service +apiVersion: v1 +metadata: + name: kube-sidecar-injector-fargate + labels: + app.kubernetes.io/name: kube-sidecar-injector-fargate +spec: + selector: + app.kubernetes.io/name: kube-sidecar-injector-fargate + ports: + - name: https + port: 8443 + targetPort: 8443 + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-sidecar-injector-fargate + labels: + app.kubernetes.io/name: kube-sidecar-injector-fargate +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-sidecar-injector-fargate + template: + metadata: + labels: + app.kubernetes.io/name: kube-sidecar-injector-fargate + spec: + serviceAccountName: kube-sidecar-injector + containers: + - name: kube-sidecar-injector-fargate + image: kube-sidecar-injector:0.0.4-dev + args: [ + "serve", + "--mutating-webhook-configuration-name", "kube-sidecar-injector-fargate", + "--service-name", "kube-sidecar-injector-fargate", + ] + ports: + - name: https + containerPort: 8443 + volumeMounts: + - name: config + mountPath: /etc/kube-sidecar-injector + readOnly: true + volumes: + - name: config + configMap: + name: kube-sidecar-injector-fargate + items: + - key: config.yaml + path: config.yaml + +--- + +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-sidecar-injector-fargate + labels: + app.kubernetes.io/name: kube-sidecar-injector-fargate +data: + config.yaml: |- + inject: + - labelSelector: + matchLabels: + app.kubernetes.io/name: dummy-injected-via-deployment + + labels: + eks.amazonaws.com/fargate-profile: default diff --git a/deploy/deployment.yaml b/deploy/deployment-node-exporter.yaml similarity index 58% rename from deploy/deployment.yaml rename to deploy/deployment-node-exporter.yaml index 2935622..dc217d8 100644 --- a/deploy/deployment.yaml +++ b/deploy/deployment-node-exporter.yaml @@ -1,25 +1,46 @@ --- +kind: Service +apiVersion: v1 +metadata: + name: kube-sidecar-injector-node-exporter + labels: + app.kubernetes.io/name: kube-sidecar-injector-node-exporter +spec: + selector: + app.kubernetes.io/name: kube-sidecar-injector-node-exporter + ports: + - name: https + port: 8443 + targetPort: 8443 + +--- + kind: Deployment apiVersion: apps/v1 metadata: - name: kube-sidecar-injector + name: kube-sidecar-injector-node-exporter labels: - app.kubernetes.io/name: kube-sidecar-injector + app.kubernetes.io/name: kube-sidecar-injector-node-exporter spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: kube-sidecar-injector + app.kubernetes.io/name: kube-sidecar-injector-node-exporter template: metadata: labels: - app.kubernetes.io/name: kube-sidecar-injector + app.kubernetes.io/name: kube-sidecar-injector-node-exporter spec: serviceAccountName: kube-sidecar-injector containers: - - name: kube-sidecar-injector - image: kube-sidecar-injector:0.0.3-dev + - name: kube-sidecar-injector-node-exporter + image: kube-sidecar-injector:0.0.4-dev + args: [ + "serve", + "--mutating-webhook-configuration-name", "kube-sidecar-injector-node-exporter", + "--service-name", "kube-sidecar-injector-node-exporter", + ] ports: - name: https containerPort: 8443 @@ -30,7 +51,7 @@ spec: volumes: - name: config configMap: - name: kube-sidecar-injector + name: kube-sidecar-injector-node-exporter items: - key: config.yaml path: config.yaml @@ -40,11 +61,18 @@ spec: kind: ConfigMap apiVersion: v1 metadata: - name: kube-sidecar-injector + name: kube-sidecar-injector-node-exporter + labels: + app.kubernetes.io/name: kube-sidecar-injector-node-exporter data: config.yaml: |- inject: - - labels: + - labelSelector: + matchExpressions: + - key: eks.amazonaws.com/fargate-profile + operator: Exists + + labels: flashbots.net/fargate-node-exporter: true containers: @@ -61,50 +89,3 @@ data: requests: cpu: 10m memory: 64Mi - - labelSelector: - matchExpressions: - - key: eks.amazonaws.com/fargate-profile - operator: Exists - ---- - -kind: Service -apiVersion: v1 -metadata: - name: kube-sidecar-injector - labels: - app.kubernetes.io/name: kube-sidecar-injector -spec: - selector: - app.kubernetes.io/name: kube-sidecar-injector - ports: - - name: https - port: 8443 - targetPort: 8443 - ---- - -kind: ServiceAccount -apiVersion: v1 -metadata: - name: kube-sidecar-injector - labels: - app.kubernetes.io/name: kube-sidecar-injector - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kube-sidecar-injector - labels: - app.kubernetes.io/name: kube-sidecar-injector -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kube-sidecar-injector -subjects: - - kind: ServiceAccount - name: kube-sidecar-injector - namespace: default diff --git a/deploy/dummy.yaml b/deploy/dummy.yaml index 31528c2..5edfe38 100644 --- a/deploy/dummy.yaml +++ b/deploy/dummy.yaml @@ -30,7 +30,7 @@ metadata: spec: containers: - - name: dummy + - name: dummy-injected image: ubuntu command: - /bin/bash @@ -78,3 +78,35 @@ spec: trap stop SIGTERM trap stop SIGINT while [[ ! -f stop ]]; do sleep 1; done + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: dummy-injected-via-deployment + labels: + app.kubernetes.io/name: dummy-injected-via-deployment +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dummy-injected-via-deployment + template: + metadata: + labels: + app.kubernetes.io/name: dummy-injected-via-deployment + spec: + containers: + - name: dummy-injected-via-deployment + image: ubuntu + command: + - /bin/bash + - -c + - |- + stop() { + touch stop + } + trap stop SIGTERM + trap stop SIGINT + while [[ ! -f stop ]]; do sleep 1; done diff --git a/patch/update_pod_labels.go b/patch/update_pod_labels.go index 7734afb..6451cae 100644 --- a/patch/update_pod_labels.go +++ b/patch/update_pod_labels.go @@ -14,7 +14,7 @@ func UpdatePodLabels( return nil, nil } - if len(pod.Annotations) == 0 { + if len(pod.Labels) == 0 { op, err := operation.Add("/metadata/labels", labels) if err != nil { return nil, err @@ -25,7 +25,7 @@ func UpdatePodLabels( res := make(json_patch.Patch, 0, len(labels)) for k, v := range labels { - if _, exists := pod.Annotations[k]; exists { + if _, exists := pod.Labels[k]; exists { op, err := operation.Replace("/metadata/labels/"+operation.Escape(k), v) if err != nil { return nil, err diff --git a/readme.md b/readme.md index 0de78b8..a99ed48 100644 --- a/readme.md +++ b/readme.md @@ -10,8 +10,13 @@ running next to it. ```yaml inject: - - labels: - flashbots.net/fargate-node-exporter: true + - labelSelector: + matchExpressions: + - key: eks.amazonaws.com/fargate-profile + operator: Exists + + labels: + flashbots.net/prometheus-node-exporter: true containers: - name: node-exporter @@ -28,8 +33,14 @@ inject: cpu: 10m memory: 64Mi - labelSelector: - matchExpressions: - - key: eks.amazonaws.com/fargate-profile - operator: Exists ``` + +### Caveats + +Single webhook configuration can me configured to apply multiple injection +rules. However, if these rules are supposed to interact somehow (for example +rule A introduces changes that rule B is supposed to act upon) then they should +be placed into _separate_ webhooks. + +See k8s webhook [reinvocation policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) +for the details. diff --git a/server/k8s.go b/server/k8s.go index cca698b..d270bea 100644 --- a/server/k8s.go +++ b/server/k8s.go @@ -44,6 +44,7 @@ func (s *Server) upsertMutatingWebhookConfiguration(ctx context.Context) error { failurePolicyIgnore := admission_registration_v1.Ignore sideEffectClassNone := admission_registration_v1.SideEffectClassNone + reinvocationPolicyIfNeeded := admission_registration_v1.IfNeededReinvocationPolicy webhooks := make([]admission_registration_v1.MutatingWebhook, 0, len(s.cfg.Inject)) for _, i := range s.cfg.Inject { @@ -61,6 +62,7 @@ func (s *Server) upsertMutatingWebhookConfiguration(ctx context.Context) error { AdmissionReviewVersions: []string{"v1", "v1beta1"}, FailurePolicy: &failurePolicyIgnore, ObjectSelector: objectSelector, + ReinvocationPolicy: &reinvocationPolicyIfNeeded, SideEffects: &sideEffectClassNone, ClientConfig: admission_registration_v1.WebhookClientConfig{ @@ -82,7 +84,7 @@ func (s *Server) upsertMutatingWebhookConfiguration(ctx context.Context) error { Rule: admission_registration_v1.Rule{ APIGroups: []string{""}, - APIVersions: []string{"v1"}, + APIVersions: []string{"v1", "v1beta1"}, Resources: []string{"pods"}, }, }}, @@ -171,12 +173,24 @@ func (s *Server) mutatePod( ) (json_patch.Patch, error) { l := logutils.LoggerFromContext(ctx) + annotation := s.cfg.K8S.ServiceName + "." + global.OrgDomain + "/" + fingerprint + if _, alreadyProcessed := pod.Annotations[annotation]; alreadyProcessed { + l.Info("Pod already was processes by this inject-configuration => skipping...", + zap.String("fingerprint", fingerprint), + zap.String("namespace", pod.Namespace), + zap.String("pod", pod.Name), + ) + return nil, nil + } + res := make(json_patch.Patch, 0) inject, exists := s.inject[fingerprint] if !exists { l.Warn("Unknown inject fingerprint => skipping...", zap.String("fingerprint", fingerprint), + zap.String("namespace", pod.Namespace), + zap.String("pod", pod.Name), ) return nil, nil } @@ -218,29 +232,26 @@ func (s *Server) mutatePod( res = append(res, p...) } - // only apply labels/annotations if at least one container was injected above - if len(res) > 0 { - { // label - p, err := patch.UpdatePodLabels(pod, inject.Labels) - if err != nil { - return nil, err - } - res = append(res, p...) + { // label + p, err := patch.UpdatePodLabels(pod, inject.Labels) + if err != nil { + return nil, err } + res = append(res, p...) + } - { // annotate - annotations := make(map[string]string, len(inject.Annotations)+1) - for k, v := range inject.Annotations { - annotations[k] = v - } - annotations[s.cfg.K8S.ServiceName+"."+global.OrgDomain+"/"+fingerprint] = time.Now().Format(time.RFC3339) + { // annotate + annotations := make(map[string]string, len(inject.Annotations)+1) + for k, v := range inject.Annotations { + annotations[k] = v + } + annotations[annotation] = time.Now().Format(time.RFC3339) - p, err := patch.UpdatePodAnnotations(pod, annotations) - if err != nil { - return nil, err - } - res = append(res, p...) + p, err := patch.UpdatePodAnnotations(pod, annotations) + if err != nil { + return nil, err } + res = append(res, p...) } return res, nil