From c9c351b2bcd4ef5558ea23298a36b7260ea131f5 Mon Sep 17 00:00:00 2001 From: Mickael Stanislas Date: Mon, 21 Oct 2024 15:42:16 +0200 Subject: [PATCH] chore: refactor admission controller --- .goreleaser.yaml | 60 ----- Makefile | 28 ++- cmd/admission-controller/certificate.go | 222 ------------------ cmd/admission-controller/main.go | 122 ---------- .../webhook-configuration.go | 105 --------- cmd/admission-controller/webhook.go | 220 ----------------- cmd/operator/main.go | 16 ++ config/webhook/manifests.yaml | 10 +- go.mod | 15 +- go.sum | 24 +- internal/controller/image_tag_mutator.go | 98 ++++++++ internal/httpserver/httpserver.go | 1 + internal/kubeclient/client.go | 1 + internal/kubeclient/image.go | 2 +- internal/kubeclient/mutating.go | 86 +++++++ internal/models/admission-controller.go | 14 ++ .../crd/kimup.cloudavenue.io_images.yaml | 12 + .../crd/kimup.cloudavenue.io_kimups.yaml | 136 +++++++++-- manifests/operator/deployment.yaml | 54 +++++ manifests/operator/kustomization.yaml | 3 + manifests/operator/service.yaml | 14 ++ manifests/operator/webhook-certificate.yaml | 21 ++ test/mocks/fakekubeclient/kubeclient.go | 4 + tools/admission-controller/main.go | 49 ++++ tools/env-dev/pod-operator.yaml | 31 +++ tools/env-dev/whoami-deployment.yaml | 48 ++++ 26 files changed, 617 insertions(+), 779 deletions(-) delete mode 100644 cmd/admission-controller/certificate.go delete mode 100644 cmd/admission-controller/main.go delete mode 100644 cmd/admission-controller/webhook-configuration.go delete mode 100644 cmd/admission-controller/webhook.go create mode 100644 internal/controller/image_tag_mutator.go create mode 100644 internal/kubeclient/mutating.go create mode 100644 internal/models/admission-controller.go create mode 100644 manifests/operator/deployment.yaml create mode 100644 manifests/operator/service.yaml create mode 100644 manifests/operator/webhook-certificate.yaml create mode 100644 tools/admission-controller/main.go create mode 100644 tools/env-dev/pod-operator.yaml create mode 100644 tools/env-dev/whoami-deployment.yaml diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7fc10bf..c5ad4b1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -37,17 +37,6 @@ builds: - arm64 env: - CGO_ENABLED=0 - - id: "kimup-admission-controller" - binary: kimup-admission-controller - main: ./cmd/admission-controller - goos: - - linux - - darwin - goarch: - - amd64 - - arm64 - env: - - CGO_ENABLED=0 dockers: # * KIMUP @@ -128,45 +117,6 @@ dockers: - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - # * KIMUP-ADMISSION-CONTROLLER - - goarch: amd64 - image_templates: - - "ghcr.io/orange-cloudavenue/{{.ProjectName}}-admission-controller:v{{ .Version }}-amd64" - dockerfile: Dockerfile - use: buildx - ids: - - kimup-admission-controller - build_flag_templates: - - --platform=linux/amd64 - - "--build-arg=BINNAME=kimup-admission-controller" - - --pull - - --label=org.opencontainers.image.title="kimup-admission-controller" - - --label=org.opencontainers.image.description="kube-image-updater-admission-controller" - - --label=org.opencontainers.image.url=https://github.com/orange-cloudavenue/kube-image-updater-admission-controller - - --label=org.opencontainers.image.source=https://github.com/orange-cloudavenue/kube-image-updater-admission-controller - - --label=org.opencontainers.image.version={{ .Version }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - - goarch: arm64 - image_templates: - - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}-arm64v8" - dockerfile: Dockerfile - use: buildx - ids: - - kimup-admission-controller - build_flag_templates: - - --platform=linux/arm64/v8 - - "--build-arg=BINNAME=kimup-admission-controller" - - --pull - - --label=org.opencontainers.image.title="kimup-admission-controller" - - --label=org.opencontainers.image.description="kube-image-updater-admission-controller" - - --label=org.opencontainers.image.url=https://github.com/orange-cloudavenue/kube-image-updater-admission-controller - - --label=org.opencontainers.image.source=https://github.com/orange-cloudavenue/kube-image-updater-admission-controller - - --label=org.opencontainers.image.version={{ .Version }} - - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - docker_manifests: # * KIMUP - name_template: "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-controller:v{{ .Version }}" @@ -187,13 +137,3 @@ docker_manifests: image_templates: - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-operator:v{{ .Version }}-amd64" - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-operator:v{{ .Version }}-arm64v8" - -# * KIMUP-ADMISSION-CONTROLLER -- name_template: "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}" - image_templates: - - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}-amd64" - - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}-arm64v8" -- name_template: "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:latest" - image_templates: - - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}-amd64" - - "ghcr.io/orange-cloudavenue/{{ .ProjectName }}-admission-controller:v{{ .Version }}-arm64v8" diff --git a/Makefile b/Makefile index c90884a..d9f064e 100644 --- a/Makefile +++ b/Makefile @@ -82,28 +82,38 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes build: manifests generate fmt vet ## Build manager binary. go build -o bin/operator cmd/operator/main.go go build -o bin/kimup cmd/kimup/* - go build -o bin/admission-controller cmd/admission-controller/* - -.PHONY: build -build-admission-controller: manifests generate fmt vet - go build -o bin/admission-controller cmd/admission-controller/* .PHONY: build-kimup build-kimup: manifests generate fmt vet go build -o bin/kimup cmd/kimup/* +.PHONY: generate-mutating-config +generate-mutating-config: ## Generate the mutating webhook configuration. + go run ./tools/admission-controller/main.go + .PHONY: run-operator run-operator: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/operator/main.go +.PHONY: run-operator-in-cluster +run-operator-in-cluster: manifests generate fmt vet ## Run a controller from your host. + kubectl -n kimup-operator delete po kimup-operator || true + kubectl create ns kimup-operator || true + kubectl apply -k manifests/crd + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml --wait=true + kubectl -n kimup-operator apply --filename="manifests/operator/role.yaml,manifests/operator/role_binding.yaml,manifests/operator/service_account.yaml,manifests/operator/webhook-certificate.yaml,manifests/operator/service.yaml" --wait=true + kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=300s + kubectl wait --for=condition=Ready certificate/kimup-webhook-serving-cert -n kimup-operator + kurun apply -f tools/env-dev/pod-operator.yaml + kubectl wait --for=condition=Ready pod/kimup-operator -n kimup-operator + kubectl apply -f tools/env-dev/whoami-deployment.yaml || true + kubectl -n kimup-operator logs kimup-operator -f + kubectl -n kimup-operator delete po kimup-operator + .PHONY: run-kimup run-kimup: manifests generate fmt vet ## Run the image updater from your host. go run ./cmd/kimup -.PHONY: run-admission-controller -run-admission-controller: manifests generate fmt vet ## Run the admission-controller from your host. - go run ./cmd/admission-controller/ - .PHONY: run-mkdocs run-mkdocs: ## Run mkdocs to serve the documentation locally. mkdocs serve diff --git a/cmd/admission-controller/certificate.go b/cmd/admission-controller/certificate.go deleted file mode 100644 index 324bc51..0000000 --- a/cmd/admission-controller/certificate.go +++ /dev/null @@ -1,222 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "context" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" - "encoding/pem" - "math/big" - "os" - "strings" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - v1 "k8s.io/client-go/applyconfigurations/admissionregistration/v1" - - "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient" -) - -// generateTLS generates a self-signed certificate for the webhook server -// and returns the certificate and the CA certificate -// The certificate is generated with the following DNS names: -// - webhookServiceName -// - webhookServiceName.webhookNamespace -// - webhookServiceName.webhookNamespace.svc -func generateTLS() (keyPair tls.Certificate, caPEM *bytes.Buffer, err error) { - // generate dns names - dnsNames := []string{ - webhookServiceName, - webhookServiceName + "." + webhookNamespace, - webhookServiceName + "." + webhookNamespace + ".svc", - // webhookServiceName + "." + webhookNamespace + ".svc" + ".cluster.local", - } - commonName := webhookServiceName + "." + webhookNamespace + ".svc" - - caPEM, certPEM, certKeyPEM, err := generateCert([]string{webhookBase}, dnsNames, commonName) - if err != nil { - return - } - - keyPair, err = tls.X509KeyPair(certPEM.Bytes(), certKeyPEM.Bytes()) - if err != nil { - return - } - return -} - -// generateCert generates a self-signed certificate with the given organizations, DNS names, and common name -// The certificate is valid for 1 year -// The certificate is signed by the CA certificate -// The CA certificate is generated with the given organizations -// it resurns the CA, certificate and private key in PEM format. -func generateCert(orgs, dnsNames []string, commonName string) (caPEM, newCertPEM, newPrivateKeyPEM *bytes.Buffer, err error) { - // init CA config - ca := &x509.Certificate{ - SerialNumber: big.NewInt(2022), - Subject: pkix.Name{Organization: orgs}, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(1, 0, 0), // expired in 1 year - IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - BasicConstraintsValid: true, - } - - // generate private key for CA - caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, nil, err - } - - // create the CA certificate - caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey) - if err != nil { - return nil, nil, nil, err - } - - // CA certificate with PEM encoded - caPEM = new(bytes.Buffer) - err = pem.Encode(caPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: caBytes, - }) - if err != nil { - return nil, nil, nil, err - } - - // print CA certificate if insideCluster is false - if !insideCluster { - writeNewCA(caPEM, manifestWebhookPath) - time.Sleep(2 * time.Second) - applyManifest(manifestWebhookPath) - } - - // new certificate config - newCert := &x509.Certificate{ - DNSNames: dnsNames, - SerialNumber: big.NewInt(1024), - Subject: pkix.Name{ - CommonName: commonName, - Organization: orgs, - }, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(1, 0, 0), // expired in 1 year - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature, - } - - // generate new private key - newPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, nil, err - } - - // sign the new certificate - newCertBytes, err := x509.CreateCertificate(rand.Reader, newCert, ca, &newPrivateKey.PublicKey, caPrivateKey) - if err != nil { - return nil, nil, nil, err - } - - // new certificate with PEM encoded - newCertPEM = new(bytes.Buffer) - err = pem.Encode(newCertPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: newCertBytes, - }) - if err != nil { - return nil, nil, nil, err - } - - // new private key with PEM encoded - newPrivateKeyPEM = new(bytes.Buffer) - err = pem.Encode(newPrivateKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(newPrivateKey), - }) - if err != nil { - return nil, nil, nil, err - } - - return caPEM, newCertPEM, newPrivateKeyPEM, nil -} - -func writeNewCA(caPEM *bytes.Buffer, filePath string) { - newCABundle := base64.StdEncoding.EncodeToString(caPEM.Bytes()) - - // Lire le fichier - file, err := os.Open(filePath) - if err != nil { - return - } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "caBundle:") { - line = " caBundle: " + "\"" + newCABundle + "\"" - } - lines = append(lines, line) - } - - if err := scanner.Err(); err != nil { - return - } - - // Écrire les modifications dans le fichier - file, err = os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - return - } - defer file.Close() - - writer := bufio.NewWriter(file) - for _, line := range lines { - _, err := writer.WriteString(line + "\n") - if err != nil { - return - } - } - writer.Flush() -} - -func applyManifest(file string) { - // read the manifest file - manifestBytes, err := os.ReadFile(file) - if err != nil { - return - } - - // decode the manifest to unstructured object - decoder := serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer() - obj := &unstructured.Unstructured{} - _, _, err = decoder.Decode(manifestBytes, nil, obj) - if err != nil { - return - } - - // convert the unstructured object to typed object - mutatingWebhookConfiguration, err := kubeclient.DecodeUnstructured[v1.MutatingWebhookConfigurationApplyConfiguration](obj) - if err != nil { - return - } - - // apply the manifest - if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Apply( - context.TODO(), - &mutatingWebhookConfiguration, - metav1.ApplyOptions{Force: true, FieldManager: "kumi-webhook"}, - ); err != nil { - return - } -} diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go deleted file mode 100644 index fa747a6..0000000 --- a/cmd/admission-controller/main.go +++ /dev/null @@ -1,122 +0,0 @@ -package main - -import ( - "context" - "crypto/tls" - "flag" - "net" - "os" - "os/signal" - "syscall" - "time" - - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - - "github.com/orange-cloudavenue/kube-image-updater/internal/httpserver" - client "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient" - "github.com/orange-cloudavenue/kube-image-updater/internal/log" - "github.com/orange-cloudavenue/kube-image-updater/internal/metrics" -) - -var ( - insideCluster bool = true // running inside k8s cluster - - webhookNamespace string = "nip.io" - webhookServiceName string = "192-168-1-30" - webhookConfigName string = "mutating-webhook-configuration" - webhookPathMutate string = "/mutate" - webhookPort string = ":8443" - webhookBase = webhookServiceName + "." + webhookNamespace - - runtimeScheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(runtimeScheme) - deserializer = codecs.UniversalDeserializer() - - kubeClient client.Interface - manifestWebhookPath string = "./examples/mutatingWebhookConfiguration.yaml" -) - -func init() { - // Init Metrics - metrics.AdmissionController() - - // webhook server running namespace (default to "default") - if os.Getenv("POD_NAMESPACE") != "" { - webhookNamespace = os.Getenv("POD_NAMESPACE") - } - // init flags - flag.StringVar(&webhookPort, "webhook-port", webhookPort, "Webhook server port.ex: :8443") - flag.StringVar(&webhookNamespace, "namespace", webhookNamespace, "Kimup Webhook Mutating namespace.") - flag.StringVar(&webhookServiceName, "service-name", webhookServiceName, "Kimup Webhook Mutating service name.") - flag.BoolVar(&insideCluster, "inside-cluster", true, "True if running inside k8s cluster.") - flag.Parse() -} - -// Start http server for webhook -func main() { - var err error - - // -- Context -- // - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // -- OS signal handling -- // - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL) - - // kubernetes golang library provide flag "kubeconfig" to specify the path to the kubeconfig file - kubeClient, err = client.New(flag.Lookup("kubeconfig").Value.String(), client.ComponentAdmissionController) - if err != nil { - log.WithError(err).Panicf("Error creating kubeclient") - } - - // * Webhook server - // generate cert for webhook - pair, caPEM, err := generateTLS() - if err != nil { - log.WithError(err).Fatal("Failed to generate TLS") - } - tlsC := &tls.Config{ - Certificates: []tls.Certificate{pair}, - MinVersion: tls.VersionTLS12, - // InsecureSkipVerify: true, //nolint:gosec - } - - // create or update the mutatingwebhookconfiguration - err = createOrUpdateMutatingWebhookConfiguration(caPEM, webhookServiceName, webhookNamespace, kubeClient) - if err != nil { - log.WithError(err).Error("Failed to create or update the mutating webhook configuration") - signalChan <- os.Interrupt - } - - // * Config the webhook server - a, waitHTTP := httpserver.Init(ctx, httpserver.WithCustomHandlerForHealth( - func() (bool, error) { - _, err := net.DialTimeout("tcp", webhookPort, 5*time.Second) - if err != nil { - return false, err - } - return true, nil - })) - - s, err := a.Add("webhook", httpserver.WithTLS(tlsC), httpserver.WithAddr(webhookPort)) - if err != nil { - log. - WithError(err). - WithFields(logrus.Fields{ - "address": webhookPort, - }).Fatal("Failed to create the server") - } - s.Config.Post(webhookPathMutate, ServeHandler) - if err := a.Run(); err != nil { - log.WithError(err).Fatal("Failed to start HTTP servers") - } - - // !-- OS signal handling --! // - <-signalChan - // cancel the context - cancel() - waitHTTP() -} diff --git a/cmd/admission-controller/webhook-configuration.go b/cmd/admission-controller/webhook-configuration.go deleted file mode 100644 index a49921b..0000000 --- a/cmd/admission-controller/webhook-configuration.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "bytes" - "context" - "reflect" - - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - client "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient" - "github.com/orange-cloudavenue/kube-image-updater/internal/log" -) - -// createOrUpdateMutatingWebhookConfiguration creates or updates the mutating webhook configuration -// for the webhook service. The CA is generated and used for the webhook. -// This function create the request to the Kubernetes API server to create or update the mutating webhook configuration. -func createOrUpdateMutatingWebhookConfiguration(caPEM *bytes.Buffer, webhookService, webhookNamespace string, k client.Interface) error { - mutatingWebhookConfigV1Client := k.AdmissionregistrationV1() - - var clientConfig admissionregistrationv1.WebhookClientConfig - switch insideCluster { - case true: - clientConfig = admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: webhookService, - Namespace: webhookNamespace, - Path: &webhookPathMutate, - }, - } - case false: - // the webhook is running outside the cluster - // Please note that the webhook service must be accessible from the Kubernetes cluster. - // Each time you change webhook service name, namespace, or port, you need to update the MutatingWebhookConfiguration - // Also you need to modifiy the manifest MutatingWebhookConfiguration to push new caPEM to allow client to trust the webhook - // The caPEM is generated and printed to the logs when the webhook starts for outside cluster - url := "https://" + webhookService + "." + webhookNamespace + webhookPort + webhookPathMutate - clientConfig = admissionregistrationv1.WebhookClientConfig{ - CABundle: caPEM.Bytes(), - URL: &url, - } - } - log.Debug("Creating or updating the mutatingwebhookconfiguration") - fail := admissionregistrationv1.Fail - sideEffect := admissionregistrationv1.SideEffectClassNone - mutatingWebhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: webhookConfigName, - }, - Webhooks: []admissionregistrationv1.MutatingWebhook{{ - Name: webhookService + "." + webhookNamespace, - AdmissionReviewVersions: []string{"v1", "v1beta1"}, - SideEffects: &sideEffect, - ClientConfig: clientConfig, - - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: []admissionregistrationv1.OperationType{ - admissionregistrationv1.Update, - admissionregistrationv1.Create, - }, - Rule: admissionregistrationv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, - // TODO - add namespace scope - // Scope: "*", - }, - }, - }, - FailurePolicy: &fail, - }}, - } - - // check if the mutatingwebhookconfiguration already exists - foundWebhookConfig, err := mutatingWebhookConfigV1Client.MutatingWebhookConfigurations().Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) - switch { - case err != nil && apierrors.IsNotFound(err): - if _, err := mutatingWebhookConfigV1Client.MutatingWebhookConfigurations().Create(context.TODO(), mutatingWebhookConfig, metav1.CreateOptions{}); err != nil { - return err - } - case err != nil: - return err - default: - // there is an existing mutatingWebhookConfiguration - if len(foundWebhookConfig.Webhooks) != len(mutatingWebhookConfig.Webhooks) || - !(foundWebhookConfig.Webhooks[0].Name == mutatingWebhookConfig.Webhooks[0].Name && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].AdmissionReviewVersions, mutatingWebhookConfig.Webhooks[0].AdmissionReviewVersions) && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].SideEffects, mutatingWebhookConfig.Webhooks[0].SideEffects) && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].FailurePolicy, mutatingWebhookConfig.Webhooks[0].FailurePolicy) && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].Rules, mutatingWebhookConfig.Webhooks[0].Rules) && - // reflect.DeepEqual(foundWebhookConfig.Webhooks[0].NamespaceSelector, mutatingWebhookConfig.Webhooks[0].NamespaceSelector) && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].ClientConfig.CABundle, mutatingWebhookConfig.Webhooks[0].ClientConfig.CABundle) && - // reflect.DeepEqual(foundWebhookConfig.Webhooks[0].ClientConfig.Service, mutatingWebhookConfig.Webhooks[0].ClientConfig.Service) && - reflect.DeepEqual(foundWebhookConfig.Webhooks[0].ClientConfig.URL, mutatingWebhookConfig.Webhooks[0].ClientConfig.URL)) { - mutatingWebhookConfig.ObjectMeta.ResourceVersion = foundWebhookConfig.ObjectMeta.ResourceVersion - if _, err := mutatingWebhookConfigV1Client.MutatingWebhookConfigurations().Update(context.TODO(), mutatingWebhookConfig, metav1.UpdateOptions{}); err != nil { - return err - } - } - } - - return nil -} diff --git a/cmd/admission-controller/webhook.go b/cmd/admission-controller/webhook.go deleted file mode 100644 index 64a350c..0000000 --- a/cmd/admission-controller/webhook.go +++ /dev/null @@ -1,220 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/sirupsen/logrus" - admissionv1 "k8s.io/api/admission/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/orange-cloudavenue/kube-image-updater/api/v1alpha1" - "github.com/orange-cloudavenue/kube-image-updater/internal/annotations" - "github.com/orange-cloudavenue/kube-image-updater/internal/log" - "github.com/orange-cloudavenue/kube-image-updater/internal/metrics" - "github.com/orange-cloudavenue/kube-image-updater/internal/patch" - "github.com/orange-cloudavenue/kube-image-updater/internal/utils" -) - -// func serveHandler -func ServeHandler(w http.ResponseWriter, r *http.Request) { - // Prometheus metrics - metrics.AdmissionController().RequestTotal.Inc() - timeAC := metrics.AdmissionController().RequestDuration.NewTimer() - defer timeAC.ObserveDuration() - - var body []byte - if r.Body != nil { - if data, err := io.ReadAll(r.Body); err == nil { - body = data - } - } - if len(body) == 0 { - // increment the total number of errors - metrics.AdmissionController().RequestErrorTotal.Inc() - - log.Error("empty body") - http.Error(w, "empty body", http.StatusBadRequest) - return - } - - // verify the content type is accurate - contentType := r.Header.Get("Content-Type") - if contentType != "application/json" { - // increment the total number of errors - metrics.AdmissionController().RequestErrorTotal.Inc() - - http.Error(w, "invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType) - return - } - - var admissionResponse *admissionv1.AdmissionResponse - ar := admissionv1.AdmissionReview{} - if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { - // increment the total number of errors - metrics.AdmissionController().RequestErrorTotal.Inc() - - log.WithError(err).Warn("Can't decode body") - admissionResponse = &admissionv1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } else { - admissionResponse = mutate(r.Context(), &ar) - } - - admissionReview := admissionv1.AdmissionReview{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "admission.k8s.io/v1", - Kind: "AdmissionReview", - }, - } - if admissionResponse != nil { - admissionReview.Response = admissionResponse - if ar.Request != nil { - admissionReview.Response.UID = ar.Request.UID - } - } - - resp, err := json.Marshal(admissionReview) - if err != nil { - // increment the total number of errors - metrics.AdmissionController().RequestErrorTotal.Inc() - - http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) - } - if _, err := w.Write(resp); err != nil { - // increment the total number of errors - metrics.AdmissionController().RequestErrorTotal.Inc() - - http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) - } -} - -// func mutate the request -func mutate(ctx context.Context, ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse { - req := ar.Request - var pod corev1.Pod - if err := json.Unmarshal(req.Object.Raw, &pod); err != nil { - return &admissionv1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - log.WithFields(logrus.Fields{ - "Kind": req.Kind, - "Namespace": req.Namespace, - "Name": req.Name, - "UID": req.UID, - "Operation": req.Operation, - "UserInfo": req.UserInfo, - }).Info("AdmissionReview") - - // create patch - patchBytes, err := createPatch(ctx, &pod) - if err != nil { - return &admissionv1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - return &admissionv1.AdmissionResponse{ - Allowed: true, - Patch: patchBytes, - PatchType: func() *admissionv1.PatchType { - pt := admissionv1.PatchTypeJSONPatch - return &pt - }(), - } -} - -// create mutation patch for pod. -func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) { - // Metrics - increment the total number of patch - metrics.AdmissionController().PatchTotal.Inc() - timePatch := metrics.AdmissionController().PatchDuration.NewTimer() - defer timePatch.ObserveDuration() - - var err error - // find annotation enabled - an := annotations.New(ctx, pod) - if !an.Enabled().Get() { - // increment the total number of errors - metrics.AdmissionController().PatchErrorTotal.Inc() - - return nil, fmt.Errorf("annotation not enabled") - } - - // var patch []patchOperation - p := patch.NewBuilder() - - log. - WithFields(logrus.Fields{ - "Namespace": pod.Namespace, - "Name": pod.Name, - }).Info("Generate Patch") - - for i, container := range pod.Spec.Containers { - imageP := utils.ImageParser(container.Image) - - // TODO Why is this not used? Annotation is never set. - crdName, _ := an.Images().Get(imageP.GetImageWithoutTag()) - - // If crdName is empty, it means that we need to find it - var image v1alpha1.Image - if crdName == "" { - // find the image associated with the pod - image, err = kubeClient.Image().Find(ctx, pod.Namespace, imageP.GetImageWithoutTag()) - if err != nil { - // increment the total number of errors - metrics.AdmissionController().PatchErrorTotal.Inc() - - log. - WithFields(logrus.Fields{ - "Namespace": pod.Namespace, - "Name": pod.Name, - "Container": container.Name, - "ContainerImage": imageP.GetImageWithoutTag(), - }). - WithError(err).Error("Failed to find kind Image") - continue - } - } else { - image, err = kubeClient.Image().Get(ctx, pod.Namespace, crdName) - if err != nil { - // increment the total number of errors - metrics.AdmissionController().PatchErrorTotal.Inc() - - log. - WithFields(logrus.Fields{ - "Namespace": pod.Namespace, - "Name": pod.Name, - "Container": container.Name, - "ContainerImage": crdName, - }).WithError(err).Error("Failed to get kind Image") - continue - } - } - - // Set the image to the pod - if image.ImageIsEqual(container.Image) { - p.AddPatch(patch.OpReplace, fmt.Sprintf("/spec/containers/%d/image", i), image.GetImageWithTag()) - } - - // Annotations - an.Containers().Set(container.Name, image.Name) - } - - // update the annotation - p.AddRawPatches(an.Containers().BuildPatches()) - - return p.Generate() -} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 83d5b99..589abee 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -32,6 +32,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" kimupv1alpha1 "github.com/orange-cloudavenue/kube-image-updater/api/v1alpha1" "github.com/orange-cloudavenue/kube-image-updater/internal/controller" @@ -68,6 +69,8 @@ func main() { ctrl.SetLogger(logrusr.New(log.GetLogger())) + webhook := webhook.NewServer(webhook.Options{}) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ @@ -78,6 +81,7 @@ func main() { HealthProbeBindAddress: "0", // disable health probe service LeaderElection: enableLeaderElection, LeaderElectionID: "71be4586.cloudavenue.io", + WebhookServer: webhook, }) if err != nil { log.WithError(err).Error("unable to start manager") @@ -90,6 +94,18 @@ func main() { c <- syscall.SIGINT } + // ! Mutator + + if err := (&controller.ImageTagMutator{ + Client: mgr.GetClient(), + KubeAPIClient: kubeAPIClient, + }).SetupWebhookWithManager(mgr); err != nil { + log.WithError(err).Error("unable to create webhook", "webhook", "ImageTagMutator") + c <- syscall.SIGINT + } + + // ! Reconcilers + if err = (&controller.ImageReconciler{ Client: mgr.GetClient(), KubeAPIClient: kubeAPIClient, diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 8a3d6f2..0c88da3 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -10,17 +10,17 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-kimup-cloudavenue-io-v1alpha1-image + path: /mutate/image-tag failurePolicy: Fail - name: mimage.kb.io + name: mutator.kimup.cloudavenue.io rules: - apiGroups: - - kimup.cloudavenue.io + - "" apiVersions: - - v1alpha1 + - v1 operations: - CREATE - UPDATE resources: - - images + - pods sideEffects: None diff --git a/go.mod b/go.mod index b021c8e..8c7a766 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ //nolint:gomoddirectives module github.com/orange-cloudavenue/kube-image-updater -go 1.22.0 +go 1.23.0 + +toolchain go1.23.1 // Temporary fix : https://github.com/crazy-max/diun/pull/1235 replace github.com/distribution/reference => github.com/distribution/reference v0.5.0 @@ -51,7 +53,8 @@ require ( github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -95,7 +98,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect @@ -111,7 +114,7 @@ require ( golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.34.2 // indirect @@ -121,8 +124,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 // indirect + k8s.io/utils v0.0.0-20240821151609-f90d01438635 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 3274c4b..0ca383c 100644 --- a/go.sum +++ b/go.sum @@ -58,10 +58,10 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= @@ -187,8 +187,8 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/reugn/go-quartz v0.13.0 h1:0eMxvj28Qu1npIDdN9Mzg9hwyksGH6XJt4Cz0QB8EUk= @@ -268,8 +268,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -310,10 +310,10 @@ k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auTjmmmuZ3O0LFMN6S0lPPhI= +k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= +k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/internal/controller/image_tag_mutator.go b/internal/controller/image_tag_mutator.go new file mode 100644 index 0000000..4743ef2 --- /dev/null +++ b/internal/controller/image_tag_mutator.go @@ -0,0 +1,98 @@ +package controller + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/orange-cloudavenue/kube-image-updater/api/v1alpha1" + "github.com/orange-cloudavenue/kube-image-updater/internal/annotations" + "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient" + "github.com/orange-cloudavenue/kube-image-updater/internal/metrics" + "github.com/orange-cloudavenue/kube-image-updater/internal/utils" +) + +func (i *ImageTagMutator) SetupWebhookWithManager(mgr ctrl.Manager) error { + wh := admission.WithCustomDefaulter(mgr.GetScheme(), &corev1.Pod{}, i) + wh.WithRecoverPanic(true) + mgr.GetWebhookServer().Register("/mutate/image-tag", &webhook.Admission{Handler: wh}) + return nil +} + +// +kubebuilder:webhook:path=/mutate/image-tag,mutating=true,failurePolicy=fail,groups="",resources=pods,sideEffects=None,verbs=create;update,versions=v1,name=mutator.kimup.cloudavenue.io,admissionReviewVersions=v1 + +var _ admission.CustomDefaulter = &ImageTagMutator{} + +// podAnnotator annotates Pods +type ImageTagMutator struct { + client.Client + KubeAPIClient *kubeclient.Client +} + +func (i *ImageTagMutator) Default(ctx context.Context, obj runtime.Object) error { + log := logf.FromContext(ctx) + pod, ok := obj.(*corev1.Pod) + if !ok { + return fmt.Errorf("expected a Pod but got a %T", obj) + } + + an := annotations.New(ctx, pod) + if !an.Enabled().Get() { + // increment the total number of errors + metrics.AdmissionController().PatchErrorTotal.Inc() + log.Info(fmt.Sprintf("annotation not enabled for pod %s/%s. Ignore it", pod.Namespace, pod.Name)) + // Return nil because we don't want to mutate the pod + return nil + } + + for _, container := range pod.Spec.Containers { + imageP := utils.ImageParser(container.Image) + + // TODO Why is this not used? Annotation is never set. + crdName, _ := an.Images().Get(imageP.GetImageWithoutTag()) + + // If crdName is empty, it means that we need to find it + var ( + image v1alpha1.Image + err error + ) + + if crdName == "" { + // find the image associated with the pod + image, err = i.KubeAPIClient.Image().Find(ctx, pod.Namespace, imageP.GetImageWithoutTag()) + if err != nil { + // increment the total number of errors + metrics.AdmissionController().PatchErrorTotal.Inc() + + log.Error(err, "Failed to find kind Image") + continue + } + } else { + image, err = i.KubeAPIClient.Image().Get(ctx, pod.Namespace, crdName) + if err != nil { + // increment the total number of errors + metrics.AdmissionController().PatchErrorTotal.Inc() + + log.Error(err, "Failed to get kind Image") + continue + } + } + + container.Image = image.GetImageWithTag() + // // Set the image to the pod + // if image.ImageIsEqual(container.Image) { + // } + + // Annotations + // an.Containers().Set(container.Name, image.Name) + } + + return nil +} diff --git a/internal/httpserver/httpserver.go b/internal/httpserver/httpserver.go index b491eb2..3fcf12a 100644 --- a/internal/httpserver/httpserver.go +++ b/internal/httpserver/httpserver.go @@ -150,6 +150,7 @@ func (a *app) createMetrics() *server { func (a *app) new(opts ...Option) *server { // create a new router r := chi.NewRouter() + r.Use(middleware.Recoverer) r.Use(middleware.Logger) // create a new server with default parameters diff --git a/internal/kubeclient/client.go b/internal/kubeclient/client.go index f92d1b4..d4dcf58 100644 --- a/internal/kubeclient/client.go +++ b/internal/kubeclient/client.go @@ -43,6 +43,7 @@ type ( InterfaceKimup interface { Image() *ImageObj Alert() *AlertObj + AdmissionController() *AdmissionControllerObj } component string diff --git a/internal/kubeclient/image.go b/internal/kubeclient/image.go index ae69c6f..4428824 100644 --- a/internal/kubeclient/image.go +++ b/internal/kubeclient/image.go @@ -196,7 +196,7 @@ func (i *ImageObj) UpdateStatus(ctx context.Context, image v1alpha1.Image) error return err } - _, err = i.imageClient.Namespace(image.Namespace).UpdateStatus(ctx, u, v1.UpdateOptions{}) + _, err = i.imageClient.Namespace(image.Namespace).UpdateStatus(ctx, u, metav1.UpdateOptions{}) if err != nil { return err } diff --git a/internal/kubeclient/mutating.go b/internal/kubeclient/mutating.go new file mode 100644 index 0000000..684f38c --- /dev/null +++ b/internal/kubeclient/mutating.go @@ -0,0 +1,86 @@ +package kubeclient + +import ( + "context" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/orange-cloudavenue/kube-image-updater/internal/annotations" + "github.com/orange-cloudavenue/kube-image-updater/internal/utils" +) + +type ( + AdmissionControllerObj struct { + InterfaceKubernetes + } +) + +// AdmissionController returns an AdmissionController object +func (c *Client) AdmissionController() *AdmissionControllerObj { + return NewAdmissionController(c) +} + +func NewAdmissionController(k InterfaceKubernetes) *AdmissionControllerObj { + return &AdmissionControllerObj{ + InterfaceKubernetes: k, + } +} + +func (a *AdmissionControllerObj) GetMutatingConfiguration(ctx context.Context, name string) (*admissionregistrationv1.MutatingWebhookConfiguration, error) { + return a.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, name, metav1.GetOptions{}) +} + +func (a *AdmissionControllerObj) CreateOrUpdateMutatingConfiguration(ctx context.Context, name string, svc admissionregistrationv1.ServiceReference, policy admissionregistrationv1.FailurePolicyType) (*admissionregistrationv1.MutatingWebhookConfiguration, error) { + mutatingWebhookConfig := a.buildMutatingConfiguration(name, svc, policy) + if _, err := a.GetMutatingConfiguration(ctx, name); err != nil { + if apierrors.IsNotFound(err) { + return a.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, mutatingWebhookConfig, metav1.CreateOptions{}) + } + return nil, err + } + + return a.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, mutatingWebhookConfig, metav1.UpdateOptions{}) +} + +func (a *AdmissionControllerObj) buildMutatingConfiguration(name string, svc admissionregistrationv1.ServiceReference, policy admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.MutatingWebhookConfiguration { + return &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + "cert-manager.io/inject-ca-from": "kimup-operator/kimup-webhook-serving-cert", + }, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{{ + Name: "image-tag.kimup.cloudavenue.io", + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + SideEffects: utils.ToPTR(admissionregistrationv1.SideEffectClassNone), + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &svc, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Update, + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + Scope: utils.ToPTR(admissionregistrationv1.NamespacedScope), + }, + }, + }, + NamespaceSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: string(annotations.KeyEnabled), + Operator: metav1.LabelSelectorOpIn, + Values: []string{"true", "yes"}, + }}, + }, + FailurePolicy: utils.ToPTR(policy), + }}, + } +} diff --git a/internal/models/admission-controller.go b/internal/models/admission-controller.go new file mode 100644 index 0000000..71d578d --- /dev/null +++ b/internal/models/admission-controller.go @@ -0,0 +1,14 @@ +package models + +import "fmt" + +var ( + AdmissionControllerDefaultPort int32 = 9099 + AdmissionControllerDefaultAddr = fmt.Sprintf(":%d", AdmissionControllerDefaultPort) + + AdmissionControllerMutatingWebhookConfigurationName = "kimup-admission-controller-mutating" + AdmissionControllerMutatingWebhookName = "image-tag.kimup.io" + AdmissionControllerServiceName = AdmissionControllerMutatingWebhookConfigurationName + + AdmissionControllerWebhookPathMutateImageTag = "/mutate/image-tag" +) diff --git a/manifests/crd/kimup.cloudavenue.io_images.yaml b/manifests/crd/kimup.cloudavenue.io_images.yaml index b3e6fd5..12355ea 100644 --- a/manifests/crd/kimup.cloudavenue.io_images.yaml +++ b/manifests/crd/kimup.cloudavenue.io_images.yaml @@ -21,6 +21,12 @@ spec: - jsonPath: .status.tag name: Tag type: string + - jsonPath: .status.result + name: Last-Result + type: string + - jsonPath: .status.time + name: Last-Sync + type: date name: v1alpha1 schema: openAPIV3Schema: @@ -213,13 +219,19 @@ spec: status: description: ImageStatus defines the observed state of Image properties: + result: + type: string tag: description: |- INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "make" to regenerate code after modifying this file type: string + time: + type: string required: + - result - tag + - time type: object type: object served: true diff --git a/manifests/crd/kimup.cloudavenue.io_kimups.yaml b/manifests/crd/kimup.cloudavenue.io_kimups.yaml index 33cc211..c590855 100644 --- a/manifests/crd/kimup.cloudavenue.io_kimups.yaml +++ b/manifests/crd/kimup.cloudavenue.io_kimups.yaml @@ -24,7 +24,8 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: Kimup is the Schema for the kimups API + description: Kimup is the Schema for the kimups API. Permit to manage the + Kimup instances. (Controller and AdmissionController) properties: apiVersion: description: |- @@ -44,12 +45,16 @@ spec: metadata: type: object spec: - description: KimupSpec defines the desired state of Kimup + description: Spec defines the desired state of Kimup properties: admissionController: + description: AdmissionController is a map of settings that will be + used to configure the admissionController. If not set, the admissionController + will not be deployed. properties: affinity: - description: Affinity is a group of affinity scheduling rules. + description: Affinity is a map of affinity settings that will + be added to the Kimup pods. properties: nodeAffinity: description: Describes node affinity scheduling rules for @@ -978,6 +983,8 @@ spec: annotations: additionalProperties: type: string + description: Annotations is a key value map that will be added + to the Kimup pods. type: object deploymentType: default: Deployment @@ -986,6 +993,8 @@ spec: - DaemonSet type: string env: + description: Env is a list of key value pairs that will be added + to the Kimup pods. items: description: EnvVar represents an environment variable present in a Container. @@ -1107,56 +1116,95 @@ spec: healthz: default: enabled: true + description: Healthz is a map of settings that will be used to + configure the healthz probe. If not set, the probe will be enabled. properties: enabled: default: true + description: Enabled is a boolean that enables or disables + the probe. If not set, the probe will be enabled. type: boolean path: + description: Path is the path where the probe will be exposed. + If not set, the default path will be used. See https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. type: string port: + description: Port is the port number where the probe will + be exposed. If not set, the default port will be used. See + https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. format: int32 type: integer type: object image: + description: Image of the Kimup container. If not set, the default + image will be used. type: string labels: additionalProperties: type: string + description: Labels is a key value map that will be added to the + Kimup pods. type: object logLevel: default: info + description: LogLevel is a string that will be used to configure + the log level of the Kimup instance. If not set, the info log + level will be used. enum: - debug - info - warn - error + - fatal + - panic + - trace type: string metrics: default: enabled: true + description: Metrics is a map of settings that will be used to + configure the metrics probe. If not set, the probe will be enabled. properties: enabled: default: true + description: Enabled is a boolean that enables or disables + the probe. If not set, the probe will be enabled. type: boolean path: + description: Path is the path where the probe will be exposed. + If not set, the default path will be used. See https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. type: string port: + description: Port is the port number where the probe will + be exposed. If not set, the default port will be used. See + https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. format: int32 type: integer type: object + name: + description: The name of the Kimup instance in the suffix of the + resource names. + type: string nodeSelector: additionalProperties: type: string + description: NodeSelector is a map of node selector settings that + will be added to the Kimup pods. type: object priorityClassName: + description: PriorityClassName is the name of the priority class + that will be used by the Kimup pods. type: string replicas: default: 3 + description: Replicas is the number of replicas that will be used + by the admissionController deployment. If not set, 3 replicas + will be used. (Only for Deployment) format: int32 type: integer resources: - description: ResourceRequirements describes the compute resource - requirements. + description: Resources is a map of resource requirements that + will be added to the Kimup pods. properties: claims: description: |- @@ -1216,8 +1264,12 @@ spec: type: object serviceAccountName: default: kimup + description: ServiceAccountName is the name of the service account + that will be used by the Kimup pods. type: string tolerations: + description: Tolerations is a list of tolerations that will be + added to the Kimup pods. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -1256,6 +1308,8 @@ spec: type: object type: array topologySpreadConstraints: + description: TopologySpreadConstraints is a list of constraints + that will be added to the Kimup pods. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -1430,11 +1484,17 @@ spec: - whenUnsatisfiable type: object type: array + required: + - name type: object controller: + description: Controller is a map of settings that will be used to + configure the controller. If not set, the controller will not be + deployed. properties: affinity: - description: Affinity is a group of affinity scheduling rules. + description: Affinity is a map of affinity settings that will + be added to the Kimup pods. properties: nodeAffinity: description: Describes node affinity scheduling rules for @@ -2363,8 +2423,12 @@ spec: annotations: additionalProperties: type: string + description: Annotations is a key value map that will be added + to the Kimup pods. type: object env: + description: Env is a list of key value pairs that will be added + to the Kimup pods. items: description: EnvVar represents an environment variable present in a Container. @@ -2486,52 +2550,88 @@ spec: healthz: default: enabled: true + description: Healthz is a map of settings that will be used to + configure the healthz probe. If not set, the probe will be enabled. properties: enabled: default: true + description: Enabled is a boolean that enables or disables + the probe. If not set, the probe will be enabled. type: boolean path: + description: Path is the path where the probe will be exposed. + If not set, the default path will be used. See https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. type: string port: + description: Port is the port number where the probe will + be exposed. If not set, the default port will be used. See + https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. format: int32 type: integer type: object image: + description: Image of the Kimup container. If not set, the default + image will be used. type: string labels: additionalProperties: type: string + description: Labels is a key value map that will be added to the + Kimup pods. type: object logLevel: default: info + description: LogLevel is a string that will be used to configure + the log level of the Kimup instance. If not set, the info log + level will be used. enum: - debug - info - warn - error + - fatal + - panic + - trace type: string metrics: default: enabled: true + description: Metrics is a map of settings that will be used to + configure the metrics probe. If not set, the probe will be enabled. properties: enabled: default: true + description: Enabled is a boolean that enables or disables + the probe. If not set, the probe will be enabled. type: boolean path: + description: Path is the path where the probe will be exposed. + If not set, the default path will be used. See https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. type: string port: + description: Port is the port number where the probe will + be exposed. If not set, the default port will be used. See + https://pkg.go.dev/github.com/orange-cloudavenue/kube-image-updater@v0.0.1/internal/models#pkg-variables. format: int32 type: integer type: object + name: + description: The name of the Kimup instance in the suffix of the + resource names. + type: string nodeSelector: additionalProperties: type: string + description: NodeSelector is a map of node selector settings that + will be added to the Kimup pods. type: object priorityClassName: + description: PriorityClassName is the name of the priority class + that will be used by the Kimup pods. type: string resources: - description: ResourceRequirements describes the compute resource - requirements. + description: Resources is a map of resource requirements that + will be added to the Kimup pods. properties: claims: description: |- @@ -2591,8 +2691,12 @@ spec: type: object serviceAccountName: default: kimup + description: ServiceAccountName is the name of the service account + that will be used by the Kimup pods. type: string tolerations: + description: Tolerations is a list of tolerations that will be + added to the Kimup pods. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -2631,6 +2735,8 @@ spec: type: object type: array topologySpreadConstraints: + description: TopologySpreadConstraints is a list of constraints + that will be added to the Kimup pods. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. @@ -2805,17 +2911,16 @@ spec: - whenUnsatisfiable type: object type: array + required: + - name type: object type: object status: - description: KimupStatus defines the observed state of Kimup + description: Status defines the observed state of Kimup properties: admissionController: + description: AdmissionController status properties: - isRollingUpdate: - description: IsRollingUpdate is true if the kimup instance is - being updated - type: boolean state: description: |- Status of the Kimup Instance @@ -2825,11 +2930,8 @@ spec: type: string type: object controller: + description: Controller status properties: - isRollingUpdate: - description: IsRollingUpdate is true if the kimup instance is - being updated - type: boolean state: description: |- Status of the Kimup Instance diff --git a/manifests/operator/deployment.yaml b/manifests/operator/deployment.yaml new file mode 100644 index 0000000..387bbb4 --- /dev/null +++ b/manifests/operator/deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kimup-operator + namespace: kimup-operator + labels: + app.kubernetes.io/name: kube-image-updater + app.kubernetes.io/instance: kimup-operator + app.kubernetes.io/component: controller +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: kube-image-updater + app.kubernetes.io/instance: kimup-operator + app.kubernetes.io/component: controller + template: + metadata: + labels: + app.kubernetes.io/name: kube-image-updater + app.kubernetes.io/instance: kimup-operator + app.kubernetes.io/component: controller + spec: + serviceAccountName: kimup + containers: + - name: operator + image: "ghcr.io/orange-cloudavenue/kimup-operator:latest" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-cert + readOnly: true + # readinessProbe: + # httpGet: + # path: /readyz + # port: 8081 + # livenessProbe: + # httpGet: + # path: /healthz + # port: 8081 + # resources: + + volumes: + - name: webhook-cert + secret: + defaultMode: 420 + secretName: kimup-webhook-serving-cert diff --git a/manifests/operator/kustomization.yaml b/manifests/operator/kustomization.yaml index 5700af7..734e3bc 100644 --- a/manifests/operator/kustomization.yaml +++ b/manifests/operator/kustomization.yaml @@ -6,3 +6,6 @@ resources: - role.yaml - role_binding.yaml - service_account.yaml + - webhook-certificate.yaml + - deployment.yaml + - service.yaml \ No newline at end of file diff --git a/manifests/operator/service.yaml b/manifests/operator/service.yaml new file mode 100644 index 0000000..1f1df84 --- /dev/null +++ b/manifests/operator/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mutator + namespace: kimup-operator + labels: + app.kubernetes.io/name: kube-image-updater +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app.kubernetes.io/name: kube-image-updater + app.kubernetes.io/instance: kimup-operator \ No newline at end of file diff --git a/manifests/operator/webhook-certificate.yaml b/manifests/operator/webhook-certificate.yaml new file mode 100644 index 0000000..7e49fb8 --- /dev/null +++ b/manifests/operator/webhook-certificate.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kimup-webhook-serving-cert + namespace: kimup-operator +spec: + dnsNames: + - mutator.kimup-operator.svc + - mutator.kimup-operator.svc.cluster.local + secretName: kimup-webhook-serving-cert + issuerRef: + kind: Issuer + name: kimup-selfsigned-issuer +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: kimup-selfsigned-issuer + namespace: kimup-operator +spec: + selfSigned: {} \ No newline at end of file diff --git a/test/mocks/fakekubeclient/kubeclient.go b/test/mocks/fakekubeclient/kubeclient.go index 721351c..3e88aab 100644 --- a/test/mocks/fakekubeclient/kubeclient.go +++ b/test/mocks/fakekubeclient/kubeclient.go @@ -56,6 +56,10 @@ func (f *FakeKubeClient) Alert() *kubeclient.AlertObj { return kubeclient.NewAlert(f) } +func (f *FakeKubeClient) AdmissionController() *kubeclient.AdmissionControllerObj { + return kubeclient.NewAdmissionController(f) +} + func (f *FakeKubeClient) CreateFakeImage(image v1alpha1.Image) error { u, err := kubeclient.EncodeUnstructured(image) if err != nil { diff --git a/tools/admission-controller/main.go b/tools/admission-controller/main.go new file mode 100644 index 0000000..59bf5a5 --- /dev/null +++ b/tools/admission-controller/main.go @@ -0,0 +1,49 @@ +// This tool is used to create mutating configuration for the admission controller webhook. + +package main + +import ( + "context" + "flag" + "os" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + + "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient" + "github.com/orange-cloudavenue/kube-image-updater/internal/log" + "github.com/orange-cloudavenue/kube-image-updater/internal/models" +) + +func main() { + kubeconfig := flag.Lookup("kubeconfig").Value.String() + + if kubeconfig == "" { + // Get home directory + home, err := os.UserHomeDir() + if err != nil { + panic(err) + } + + kubeconfig = home + "/.kube/config" + } + + // kubernetes golang library provide flag "kubeconfig" to specify the path to the kubeconfig file + k, err := kubeclient.New(kubeconfig, kubeclient.ComponentOperator) + if err != nil { + log.WithError(err).Panic("Error creating kubeclient") + } + + _, err = k.AdmissionController().CreateOrUpdateMutatingConfiguration( + context.Background(), + models.AdmissionControllerMutatingWebhookConfigurationName, + admissionregistrationv1.ServiceReference{ + Name: "mutator", + Namespace: "kimup-operator", + Path: &models.AdmissionControllerWebhookPathMutateImageTag, + }, + admissionregistrationv1.Fail, + ) + if err != nil { + log.WithError(err).Panic("Error creating or updating mutating configuration") + } +} diff --git a/tools/env-dev/pod-operator.yaml b/tools/env-dev/pod-operator.yaml new file mode 100644 index 0000000..952d3ee --- /dev/null +++ b/tools/env-dev/pod-operator.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: kimup-operator + app.kubernetes.io/name: kube-image-updater + name: kimup-operator + namespace: kimup-operator +spec: + containers: + - image: kurun://cmd/operator/main.go + name: operator + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-cert + readOnly: true + serviceAccount: kimup + serviceAccountName: kimup + volumes: + - name: webhook-cert + secret: + defaultMode: 420 + secretName: kimup-webhook-serving-cert \ No newline at end of file diff --git a/tools/env-dev/whoami-deployment.yaml b/tools/env-dev/whoami-deployment.yaml new file mode 100644 index 0000000..bf25f49 --- /dev/null +++ b/tools/env-dev/whoami-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: dev-kube-image-updater +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: whoami + namespace: dev-kube-image-updater + labels: + app.kubernetes.io/name: whoami +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: whoami + template: + metadata: + annotations: + kimup.cloudavenue.io/enabled: "true" + labels: + app.kubernetes.io/name: whoami + spec: + containers: + - name: whoami + image: "traefik/whoami:latest" + imagePullPolicy: IfNotPresent + +--- +apiVersion: kimup.cloudavenue.io/v1alpha1 +kind: Image +metadata: + labels: + app.kubernetes.io/name: whoami + name: traefik-whoami + namespace: dev-kube-image-updater +spec: + image: traefik/whoami + baseTag: v1.9.0 + triggers: + - type: crontab + value: "00 00 */12 * * *" + rules: + - name: Automatic update semver patch + type: semver-patch + actions: + - type: apply