diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index eef9850..e052970 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -14,13 +14,14 @@ jobs: integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Check secret id: checksecret - uses: oliverbaehler/github-actions/exists@8dfd42735c85f6c58d5d4d6f3232cd0e39d1fe73 # v0.1.0 + uses: peak-scale/github-actions/exists@38322faabccd75abfa581c435e367d446b6d2c3b # v0.1.0 with: value: ${{ secrets.CODECOV_TOKEN }} - - uses: actions/setup-go@v4 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 with: go-version: '1.19' - name: Run integration tests diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index 461a8e3..0000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Build images -permissions: {} - -on: - push: - branches: - - '*' - pull_request: - branches: [ "master", "main" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build-images: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: ko build - run: VERSION=${{ github.ref_name }} REPOSITORY=${GITHUB_REPOSITORY} make ko-build-all - - name: Trivy Scan Image - uses: aquasecurity/trivy-action@22d2755f774d925b191a185b74e782a4b0638a41 # v0.15.0 - with: - scan-type: 'fs' - ignore-unfixed: true - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8892b2e..2f0dd02 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,7 +37,7 @@ jobs: uses: sigstore/cosign-installer@11086d25041f77fe8fe7b9ea4e48e3b9192b8f19 # v3.1.2 - name: Publish with KO id: publish - uses: oliverbaehler/github-actions/ko-publish-image@dev # v0.1.0 + uses: peak-scale/github-actions/make-ko-publish@38322faabccd75abfa581c435e367d446b6d2c3b # v0.1.0 with: makefile-target: ko-publish-all registry: ghcr.io diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml index 1bbd3fd..38efdbb 100644 --- a/.github/workflows/helm-publish.yml +++ b/.github/workflows/helm-publish.yml @@ -24,7 +24,7 @@ jobs: echo "version=$(echo $VERSION)" >> $GITHUB_OUTPUT - name: Helm | Publish id: helm_publish - uses: oliverbaehler/github-actions/helm-oci-chart@dev + uses: peak-scale/github-actions/helm-oci-chart@38322faabccd75abfa581c435e367d446b6d2c3b # v0.1.0 with: registry: ghcr.io repository: ${{ github.repository_owner }}/charts diff --git a/cmd/main.go b/cmd/main.go index 191c64a..f345f12 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "log" "os" @@ -10,11 +9,14 @@ import ( "github.com/go-logr/logr" "github.com/go-logr/stdr" "github.com/spf13/cobra" + _ "go.uber.org/automaxprocs" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" crlog "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) type rootCmdFlags struct { @@ -24,16 +26,23 @@ type rootCmdFlags struct { ingressClass string // for identifying objects on parent cluster identifier string - // Kubeconfig for parent cluster - kubeconfig string // Binary log level logLevel int // Ingress class on loadbalancer cluster - targetIngressClass string - targetNamespace string - targetKubeconfig string + targetIngressClass string + targetNamespace string + targetKubeconfig string + targetIssuerNamespaced bool + targetIssuerName string + metricsAddr string + enableLeaderElection bool + tlsRepsect bool } +var ( + setupLog = ctrl.Log.WithName("setup") +) + func main() { var rootLogger = stdr.NewWithOptions(log.New(os.Stderr, "", log.LstdFlags), stdr.Options{LogCaller: stdr.All}) @@ -55,8 +64,6 @@ func main() { logger := options.logger logger.Info("logging verbosity", "level", options.logLevel) - cfg := config.GetConfigOrDie() - // Load the kubeconfig from the provided file path target, err := clientcmd.BuildConfigFromFlags("", options.targetKubeconfig) if err != nil { @@ -73,28 +80,56 @@ func main() { os.Exit(1) } - mgr, err := manager.New(cfg, manager.Options{}) + manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Metrics: metricsserver.Options{ + BindAddress: options.metricsAddr, + }, + LeaderElection: options.enableLeaderElection, + LeaderElectionID: "2c123jea.buttah.cloud", + HealthProbeBindAddress: ":10080", + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { + options.Cache.Unstructured = true + return client.New(config, options) + }, + }) if err != nil { - logger.Error(err, "unable to set up manager") + logger.Error(err, "unable to start manager") os.Exit(1) } - logger.Info("propagation controller start serving") - err = controller.RegisterPropagationController(logger, mgr, - targetClient, - controller.PropagationControllerOptions{ + _ = manager.AddReadyzCheck("ping", healthz.Ping) + _ = manager.AddHealthzCheck("ping", healthz.Ping) + + ctx := ctrl.SetupSignalHandler() + + if err = (&controller.PropagationController{ + Client: manager.GetClient(), + TargetClient: targetClient, + Log: ctrl.Log.WithName("controllers").WithName("Ingress"), + Recorder: manager.GetEventRecorderFor("ingress-controller"), + Options: controller.PropagationControllerOptions{ Identifier: options.identifier, IngressClassName: options.ingressClass, TargetIngressClassName: options.targetIngressClass, ControllerClassName: options.controllerClass, TargetNamespace: options.targetNamespace, - }) - if err != nil { - return err + TargetIssuerNamespaced: options.targetIssuerNamespaced, + TargetIssuerName: options.targetIssuerName, + TLSrespect: options.tlsRepsect, + }, + }).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Ingress") + os.Exit(1) + } + + setupLog.Info("propagation manager start serving") + + if err = manager.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) } - // controller-runtime manager would graceful shutdown with signal by itself, no need to provide context - return mgr.Start(context.Background()) + return nil }, } @@ -105,7 +140,13 @@ func main() { rootCommand.PersistentFlags().StringVar(&options.identifier, "identifier", options.identifier, "propagator identifier, if multiple propagators sync to the same target namespace, this should be different for each") rootCommand.PersistentFlags().StringVar(&options.targetNamespace, "target-namespace", options.targetNamespace, "namespace on target cluster, where manifests are synced to") rootCommand.PersistentFlags().StringVar(&options.targetKubeconfig, "target-kubeconfig", options.targetKubeconfig, "namespace on target cluster, where manifests are synced to") - + rootCommand.PersistentFlags().StringVar(&options.targetIssuerName, "target-issuer-name", options.targetIssuerName, "name of issuer added as cert-manager annotation on target cluster") + rootCommand.PersistentFlags().BoolVar(&options.targetIssuerNamespaced, "target-issuer-namespaced", false, "name of issuer added as cert-manager annotation on target cluster") + rootCommand.PersistentFlags().StringVar(&options.metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") + rootCommand.PersistentFlags().BoolVar(&options.enableLeaderElection, "enable-leader-election", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + rootCommand.PersistentFlags().BoolVar(&options.tlsRepsect, "tls-respect", false, "Respect TLS Spec on ingress objects, if an issuer is defined the TLS spec is added anyway") err := rootCommand.Execute() if err != nil { panic(err) diff --git a/go.mod b/go.mod index e282ae3..853bb69 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-logr/stdr v1.2.2 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.7.0 + go.uber.org/automaxprocs v1.5.3 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 diff --git a/go.sum b/go.sum index 5c8d1a1..812f171 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= diff --git a/helm/templates/clusterrole.yaml b/helm/templates/clusterrole.yaml index b5df77f..a225939 100644 --- a/helm/templates/clusterrole.yaml +++ b/helm/templates/clusterrole.yaml @@ -5,27 +5,36 @@ metadata: labels: {{- include "helm.labels" . | nindent 4 }} rules: - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - ingresses - - ingressclasses - verbs: - - get - - list - - watch - - update - - apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update \ No newline at end of file +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - list + - update + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index d932136..ffe3d5f 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -36,6 +36,7 @@ spec: image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: + - --enable-leader-election - --identifier={{ include "controller.identifier" $ }} - --ingress-class={{ .Values.ingressClass.name }} - --controller-class={{ include "controller.value" $ }} @@ -46,12 +47,25 @@ spec: {{- with .namespace }} - --target-namespace={{ . }} {{- end }} + {{- with .issuer }} + {{- if .name }} + - --target-issuer-name={{ .name }} + {{- end }} + {{- end }} {{- end }} - --target-kubeconfig=/target-kubeconfig.yaml volumeMounts: - name: kubeconfig-volume mountPath: /target-kubeconfig.yaml subPath: {{ .Values.target.kubeconfig.secret.key }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12}} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12}} resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/helm/values.yaml b/helm/values.yaml index 43961fd..24bcbca 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -19,6 +19,12 @@ target: ingressClass: "propagated" # -- Namespaced on target namespace: "ingress-central" + # Target Issuer + issuer: + # -- Issuer name on target cluster + name: "" + # -- Whether the issuer is namespaced on target cluster + namespaced: false # -- Target Kubeconfig Secret kubeconfig: secret: @@ -60,6 +66,19 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 +# -- Configure the liveness probe using Deployment probe spec +livenessProbe: + httpGet: + path: /healthz + port: 10080 + +# -- Configure the readiness probe using Deployment probe spec +readinessProbe: + httpGet: + path: /readyz + port: 10080 + + service: type: ClusterIP port: 80 diff --git a/pkg/controller/controlled.go b/pkg/controller/controlled.go new file mode 100644 index 0000000..cba8287 --- /dev/null +++ b/pkg/controller/controlled.go @@ -0,0 +1,47 @@ +package controller + +import ( + "context" + "fmt" + + networkingv1 "k8s.io/api/networking/v1" +) + +func (i *PropagationController) isControlledByThisController(ctx context.Context, target networkingv1.Ingress) (bool, error) { + if i.Options.IngressClassName == target.GetAnnotations()[WellKnownIngressAnnotation] { + return true, nil + } + + if target.Spec.IngressClassName == nil { + return false, nil + } + + controlledIngressClassNames, err := i.listControlledIngressClasses(ctx) + if err != nil { + return false, fmt.Errorf("fetch controlled ingress classes with controller name %s", i.Options.ControllerClassName) + } + + if stringSliceContains(controlledIngressClassNames, *target.Spec.IngressClassName) { + return true, nil + } + + return false, nil +} + +func (i *PropagationController) listControlledIngressClasses(ctx context.Context) ([]string, error) { + list := networkingv1.IngressClassList{} + err := i.Client.List(ctx, &list) + if err != nil { + return nil, err + } + + var controlledNames []string + for _, ingressClass := range list.Items { + // Check if the IngressClass is controlled by the specified controller + if ingressClass.Spec.Controller == i.Options.ControllerClassName { + controlledNames = append(controlledNames, ingressClass.Name) + } + } + + return controlledNames, nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go deleted file mode 100644 index 3ba6b49..0000000 --- a/pkg/controller/controller.go +++ /dev/null @@ -1,185 +0,0 @@ -package controller - -import ( - "context" - "fmt" - - "github.com/buttahtoast/svc-ingress-propagator/pkg/propagation" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - networkingv1 "k8s.io/api/networking/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// IngressController should implement the Reconciler interface -var _ reconcile.Reconciler = &PropagationController{} - -type PropagationController struct { - logger logr.Logger - kubeClient client.Client - targetKubeClient client.Client - options PropagationControllerOptions -} - -func NewPropagationController(logger logr.Logger, kubeClient client.Client, targetKubeClient client.Client, opts PropagationControllerOptions) *PropagationController { - return &PropagationController{logger: logger, kubeClient: kubeClient, targetKubeClient: targetKubeClient, options: opts} -} - -func (i *PropagationController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - origin := networkingv1.Ingress{} - err := i.kubeClient.Get(ctx, request.NamespacedName, &origin) - if err != nil { - if apierrors.IsNotFound(err) { - return reconcile.Result{}, nil - } - return reconcile.Result{}, errors.Wrapf(err, "fetch ingress %s", request.NamespacedName) - } - - controlled, err := i.isControlledByThisController(ctx, origin) - if err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, errors.Wrapf(err, "check if ingress %s is controlled by this controller", request.NamespacedName) - } - - if !controlled { - i.logger.V(1).Info("ingress is NOT controlled by this controller", - "ingress", request.NamespacedName, - "controlled-ingress-class", i.options.IngressClassName, - "controlled-controller-class", i.options.ControllerClassName, - ) - return reconcile.Result{ - Requeue: false, - }, nil - } - - i.logger.V(1).Info("ingress is controlled by this controller", - "ingress", request.NamespacedName, - "controlled-ingress-class", i.options.IngressClassName, - "controlled-controller-class", i.options.ControllerClassName, - ) - - i.logger.Info("update propagations", "triggered-by", request.NamespacedName) - - if origin.DeletionTimestamp == nil { - err = i.attachFinalizer(ctx, *(origin.DeepCopy())) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "attach finalizer to ingress %s", request.NamespacedName) - } - } else { - if !i.hasFinalizer(ctx, origin) { - i.logger.V(1).Info("ingress is being deleted and already finillized by this controller", - "ingress", request.NamespacedName, - "controlled-ingress-class", i.options.IngressClassName, - "controlled-controller-class", i.options.ControllerClassName, - ) - return reconcile.Result{ - Requeue: false, - }, nil - } - } - - ingresses, err := i.listControlledIngresses(ctx) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "list controlled ingresses") - } - - var managedPropagations []propagation.Propagation - for _, ingress := range ingresses { - propagations, err := FromIngressToPropagation(ctx, i.logger, i.kubeClient, ingress, i.options.TargetIngressClassName, i.options.Identifier, i.options.TargetNamespace) - if err != nil { - i.logger.Info("extract propagations from ingress, skipped", "triggered-by", request.NamespacedName, "ingress", fmt.Sprintf("%s/%s", ingress.Namespace, ingress.Name), "error", err) - } - managedPropagations = append(managedPropagations, propagations) - } - i.logger.V(3).Info("all propagations", "propagations", managedPropagations) - - err = i.TargetPropagations(ctx, managedPropagations) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "put propagations") - } - - if origin.DeletionTimestamp != nil { - err = i.cleanFinalizer(ctx, origin) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "clean finalizer from ingress %s", request.NamespacedName) - } - } - - i.logger.V(3).Info("reconcile completed", "triggered-by", request.NamespacedName) - return reconcile.Result{}, nil -} - -func (i *PropagationController) isControlledByThisController(ctx context.Context, target networkingv1.Ingress) (bool, error) { - if i.options.IngressClassName == target.GetAnnotations()[WellKnownIngressAnnotation] { - return true, nil - } - - if target.Spec.IngressClassName == nil { - return false, nil - } - - controlledIngressClassNames, err := i.listControlledIngressClasses(ctx) - if err != nil { - return false, errors.Wrapf(err, "fetch controlled ingress classes with controller name %s", i.options.ControllerClassName) - } - - if stringSliceContains(controlledIngressClassNames, *target.Spec.IngressClassName) { - return true, nil - } - - return false, nil -} - -func (i *PropagationController) listControlledIngressClasses(ctx context.Context) ([]string, error) { - list := networkingv1.IngressClassList{} - err := i.kubeClient.List(ctx, &list) - if err != nil { - return nil, errors.Wrap(err, "list ingress classes") - } - - var controlledNames []string - for _, ingressClass := range list.Items { - // Check if the IngressClass is controlled by the specified controller - if ingressClass.Spec.Controller == i.options.ControllerClassName { - controlledNames = append(controlledNames, ingressClass.Name) - } - } - - return controlledNames, nil -} - -func (i *PropagationController) listControlledIngresses(ctx context.Context) ([]networkingv1.Ingress, error) { - controlledIngressClassNames, err := i.listControlledIngressClasses(ctx) - if err != nil { - return nil, errors.Wrapf(err, "fetch controlled ingress classes with controller name %s", i.options.ControllerClassName) - } - - var result []networkingv1.Ingress - list := networkingv1.IngressList{} - err = i.kubeClient.List(ctx, &list) - if err != nil { - return nil, errors.Wrap(err, "list ingresses") - } - - for _, ingress := range list.Items { - func() { - if i.options.IngressClassName == ingress.GetAnnotations()[WellKnownIngressAnnotation] { - result = append(result, ingress) - return - } - - if ingress.Spec.IngressClassName == nil { - return - } - - if stringSliceContains(controlledIngressClassNames, *ingress.Spec.IngressClassName) { - result = append(result, ingress) - return - } - }() - } - - return result, nil -} diff --git a/pkg/controller/finalizer.go b/pkg/controller/finalizer.go deleted file mode 100644 index ff7d9f8..0000000 --- a/pkg/controller/finalizer.go +++ /dev/null @@ -1,38 +0,0 @@ -package controller - -import ( - "context" - - "github.com/pkg/errors" - networkingv1 "k8s.io/api/networking/v1" -) - -const IngressControllerFinalizer = "svc-ingress-propagator.buttah.cloud/propagated-ingress" - -func (i *PropagationController) hasFinalizer(ctx context.Context, ingress networkingv1.Ingress) bool { - return stringSliceContains(ingress.Finalizers, IngressControllerFinalizer) -} - -func (i *PropagationController) attachFinalizer(ctx context.Context, ingress networkingv1.Ingress) error { - if stringSliceContains(ingress.Finalizers, IngressControllerFinalizer) { - return nil - } - ingress.Finalizers = append(ingress.Finalizers, IngressControllerFinalizer) - err := i.kubeClient.Update(ctx, &ingress) - if err != nil { - return errors.Wrapf(err, "attach finalizer for %s/%s", ingress.Namespace, ingress.Name) - } - return nil -} - -func (i *PropagationController) cleanFinalizer(ctx context.Context, ingress networkingv1.Ingress) error { - if !stringSliceContains(ingress.Finalizers, IngressControllerFinalizer) { - return nil - } - ingress.Finalizers = removeStringFromSlice(ingress.Finalizers, IngressControllerFinalizer) - err := i.kubeClient.Update(ctx, &ingress) - if err != nil { - return errors.Wrapf(err, "clean finalizer for %s/%s", ingress.Namespace, ingress.Name) - } - return nil -} diff --git a/pkg/controller/manager.go b/pkg/controller/manager.go new file mode 100644 index 0000000..4172d08 --- /dev/null +++ b/pkg/controller/manager.go @@ -0,0 +1,131 @@ +package controller + +import ( + "context" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const IngressControllerFinalizer = "svc-ingress-propagator.buttah.cloud/propagated-ingress" + +// IngressController should implement the Reconciler interface +var _ reconcile.Reconciler = &PropagationController{} + +type PropagationController struct { + Client client.Client + TargetClient client.Client + Log logr.Logger + Recorder record.EventRecorder + Options PropagationControllerOptions +} + +type PropagationControllerOptions struct { + Identifier string + IngressClassName string + ControllerClassName string + TargetIngressClassName string + TargetNamespace string + TargetIssuerName string + TargetIssuerNamespaced bool + TLSrespect bool +} + +func (i *PropagationController) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&networkingv1.Ingress{}). + Complete(i) +} + +func (i *PropagationController) Reconcile(ctx context.Context, request ctrl.Request) (res reconcile.Result, err error) { + i.Log.V(3).Info("Reconciling", + "ingress", request.NamespacedName, + ) + origin := networkingv1.Ingress{} + if err = i.Client.Get(ctx, request.NamespacedName, &origin); err != nil { + if apierrors.IsNotFound(err) { + i.Log.Error(nil, "Request object not found, could have been deleted after reconcile request") + + return reconcile.Result{}, nil + } + + i.Log.Error(err, "Error reading the object") + + return + } + + controlled, err := i.isControlledByThisController(ctx, origin) + if err != nil && !apierrors.IsNotFound(err) { + i.Log.V(3).Error(err, "check if ingress %s is controlled by this controller", request.NamespacedName) + return reconcile.Result{ + RequeueAfter: time.Second * 60, + }, nil + } + + if !controlled { + i.Log.V(5).Info("ingress is NOT controlled by this controller", + "ingress", request.NamespacedName, + "controlled-ingress-class", i.Options.IngressClassName, + "controlled-controller-class", i.Options.ControllerClassName, + ) + return reconcile.Result{ + Requeue: false, + }, nil + } + + i.Log.V(5).Info("ingress is controlled by this controller", + "ingress", request.NamespacedName, + "controlled-ingress-class", i.Options.IngressClassName, + "controlled-controller-class", i.Options.ControllerClassName, + ) + + i.Log.V(5).Info("update propagations", "triggered-by", request.NamespacedName) + + if origin.DeletionTimestamp == nil { + controllerutil.AddFinalizer(&origin, IngressControllerFinalizer) + } else { + if !controllerutil.ContainsFinalizer(&origin, IngressControllerFinalizer) { + i.Log.V(1).Info("ingress is being deleted and already finillized by this controller", + "ingress", request.NamespacedName, + "controlled-ingress-class", i.Options.IngressClassName, + "controlled-controller-class", i.Options.ControllerClassName, + ) + + return + } + } + + propagation, err := i.FromIngressToPropagation(ctx, i.Log, i.Client, origin) + if err != nil { + i.Recorder.Eventf(&origin, corev1.EventTypeWarning, "PropagationFailed", "failed to extract propagations from ingress: %s", err.Error()) + + return reconcile.Result{ + RequeueAfter: time.Second * 60, + }, nil + } + + i.Log.V(5).Info("all propagations", "propagations", propagation.Ingress) + err = i.TargetPropagation(ctx, propagation) + if err != nil { + i.Log.Error(err, "put propagations") + + return reconcile.Result{ + RequeueAfter: time.Second * 60, + }, nil + } + + if origin.DeletionTimestamp != nil { + controllerutil.RemoveFinalizer(&origin, IngressControllerFinalizer) + } + + i.Log.V(3).Info("Reconcile completed") + return +} diff --git a/pkg/controller/meta.go b/pkg/controller/meta.go deleted file mode 100644 index 4245483..0000000 --- a/pkg/controller/meta.go +++ /dev/null @@ -1,7 +0,0 @@ -package controller - -const WellKnownIngressAnnotation = "kubernetes.io/ingress.class" -const MetaBase = "ingress-propagator.buttah.cloud" - -var LabelManaged = MetaBase + "/managed-by" -var LabelPropagator = MetaBase + "/propagator" diff --git a/pkg/controller/register.go b/pkg/controller/register.go deleted file mode 100644 index b4c8edf..0000000 --- a/pkg/controller/register.go +++ /dev/null @@ -1,38 +0,0 @@ -package controller - -import ( - "github.com/go-logr/logr" - networkingv1 "k8s.io/api/networking/v1" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -type PropagationControllerOptions struct { - Identifier string - IngressClassName string - ControllerClassName string - TargetIngressClassName string - TargetNamespace string -} - -func RegisterPropagationController(logger logr.Logger, mgr manager.Manager, targetKubeClient client.Client, options PropagationControllerOptions) error { - - controller := NewPropagationController(logger.WithName("ingress-propagator"), mgr.GetClient(), targetKubeClient, options) - err := builder. - ControllerManagedBy(mgr). - For(&networkingv1.Ingress{}). - Complete(controller) - - if err != nil { - logger.WithName("register-controller").Error(err, "could not register propagation controller") - return err - } - - if err != nil { - logger.WithName("register-controller").Error(err, "could not register propagation controller") - return err - } - - return nil -} diff --git a/pkg/controller/target.go b/pkg/controller/target.go index 2d1fcc9..7b5469a 100644 --- a/pkg/controller/target.go +++ b/pkg/controller/target.go @@ -2,146 +2,105 @@ package controller import ( "context" + "fmt" "github.com/buttahtoast/svc-ingress-propagator/pkg/propagation" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) -func (i *PropagationController) TargetPropagations(ctx context.Context, propagations []propagation.Propagation) error { - err := i.propagateIngress(ctx, propagations) - if err != nil { - return errors.Wrap(err, "ingress propagation") - } - - err = i.propagateEndpoint(ctx, propagations) - if err != nil { - return errors.Wrap(err, "endpoint propagation") - } - - err = i.propagateService(ctx, propagations) - if err != nil { - return errors.Wrap(err, "service propagation") +func (i *PropagationController) TargetPropagation(ctx context.Context, prop propagation.Propagation) error { + if prop.IsDeleted { + err := i.removePropagation(ctx, prop) + if err != nil { + return fmt.Errorf("remove propagation %s", err) + } + } else { + err := i.putPropagation(ctx, prop) + if err != nil { + return fmt.Errorf("put propagation %s", err) + } } - return nil } -func (i *PropagationController) propagateIngress(ctx context.Context, propagations []propagation.Propagation) error { - for _, prop := range propagations { - if prop.IsDeleted { - // Delete the ingress - err := i.targetKubeClient.Delete(ctx, &prop.Ingress) +func (i *PropagationController) putPropagation(ctx context.Context, prop propagation.Propagation) error { + // Try to update the ingress + err := i.TargetClient.Update(ctx, &prop.Ingress) + if err != nil { + // If error is because the resource doesn't exist, then create it + if k8serrors.IsNotFound(err) { + err = i.TargetClient.Create(ctx, &prop.Ingress) if err != nil { - return errors.Wrapf(err, "failed to delete ingress %s", prop.Ingress.Name) + return fmt.Errorf("failed to create ingress %s: %s", prop.Ingress.Name, err) } } else { - // Try to update the ingress - err := i.targetKubeClient.Update(ctx, &prop.Ingress) - if err != nil { - // If error is because the resource doesn't exist, then create it - if k8serrors.IsNotFound(err) { - err = i.targetKubeClient.Create(ctx, &prop.Ingress) - if err != nil { - return errors.Wrapf(err, "failed to create ingress %s", prop.Ingress.Name) - } - } else { - return errors.Wrapf(err, "failed to update ingress %s", prop.Ingress.Name) - } - } + return fmt.Errorf("failed to update ingress %s: %s", prop.Ingress.Name, err) } } - return nil -} - -func (i *PropagationController) propagateEndpoint(ctx context.Context, propagations []propagation.Propagation) error { - for _, prop := range propagations { - if prop.IsDeleted { - // Delete all releated endpoints - selector := labels.Set{ - LabelManaged: i.options.Identifier, - LabelPropagator: prop.Name, - } - listOptions := client.ListOptions{LabelSelector: labels.SelectorFromSet(selector), Namespace: i.options.TargetNamespace} + // Fetch the updated ingress to get the UID + updatedIngress := &v1.Ingress{} + err = i.TargetClient.Get(ctx, types.NamespacedName{Name: prop.Ingress.Name, Namespace: prop.Ingress.Namespace}, updatedIngress) + if err != nil { + return fmt.Errorf("failed to get ingress on target cluster %s: %s", prop.Ingress.Name, err) + } - var endpointsList v1.EndpointsList - err := i.targetKubeClient.List(ctx, &endpointsList, &listOptions) - if err != nil { - return errors.Wrap(err, "failed to list endpoints with label selector") - } + // OwnerReference using the Ingress UID + ownerRef := metav1.OwnerReference{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + Name: updatedIngress.Name, + UID: updatedIngress.ObjectMeta.GetUID(), + } - for _, endpoint := range endpointsList.Items { - err = i.targetKubeClient.Delete(ctx, &endpoint) + // Update Endpoints + for _, endpoint := range prop.Endpoints { + endpoint.OwnerReferences = append(endpoint.OwnerReferences, ownerRef) + err := i.TargetClient.Update(ctx, &endpoint) + if err != nil { + // If error is because the resource doesn't exist, then create it + if k8serrors.IsNotFound(err) { + err = i.TargetClient.Create(ctx, &endpoint) if err != nil { - return errors.Wrapf(err, "failed to delete endpoint %s in namespace %s", endpoint.Name, endpoint.Namespace) + return fmt.Errorf("failed to create endpoint %s in namespace %s: %s", endpoint.Name, endpoint.Namespace, err) } + } else { + return fmt.Errorf("failed to update endpoint %s in namespace %s: %s", endpoint.Name, endpoint.Namespace, err) } - } else { - for _, endpoint := range prop.Endpoints { - // Try to update the endpoint - err := i.targetKubeClient.Update(ctx, &endpoint) + } + } + for _, service := range prop.Services { + service.OwnerReferences = append(service.OwnerReferences, ownerRef) + err := i.TargetClient.Update(ctx, &service) + if err != nil { + // If error is because the resource doesn't exist, then create it + if k8serrors.IsNotFound(err) { + err = i.TargetClient.Create(ctx, &service) if err != nil { - // If error is because the resource doesn't exist, then create it - if k8serrors.IsNotFound(err) { - err = i.targetKubeClient.Create(ctx, &endpoint) - if err != nil { - return errors.Wrapf(err, "failed to create endpoint %s in namespace %s", endpoint.Name, endpoint.Namespace) - } - } else { - return errors.Wrapf(err, "failed to update endpoint %s in namespace %s", endpoint.Name, endpoint.Namespace) - } + return fmt.Errorf("failed to create service %s in namespace %s: %s", service.Name, service.Namespace, err) } + } else { + return fmt.Errorf("failed to update service %s in namespace %s: %s", service.Name, service.Namespace, err) } } } + + i.Recorder.Eventf(&prop.Origin, corev1.EventTypeNormal, "IngressPropagated", "Ingress has been propagated") return nil } -func (i *PropagationController) propagateService(ctx context.Context, propagations []propagation.Propagation) error { - for _, prop := range propagations { - if prop.IsDeleted { - selector := labels.Set{ - LabelManaged: i.options.Identifier, - LabelPropagator: prop.Name, - } - listOptions := client.ListOptions{LabelSelector: labels.SelectorFromSet(selector), Namespace: i.options.TargetNamespace} - - // List services with the label selector - var servicesList v1.ServiceList - err := i.targetKubeClient.List(ctx, &servicesList, &listOptions) - if err != nil { - return errors.Wrap(err, "failed to list services with label selector") - } - - // Delete each service from the list - for _, service := range servicesList.Items { - err = i.targetKubeClient.Delete(ctx, &service) - if err != nil { - return errors.Wrapf(err, "failed to delete service %s in namespace %s", service.Name, service.Namespace) - } - } - } else { - for _, service := range prop.Services { - // Try to update the service - err := i.targetKubeClient.Update(ctx, &service) - if err != nil { - // If error is because the resource doesn't exist, then create it - if k8serrors.IsNotFound(err) { - err = i.targetKubeClient.Create(ctx, &service) - if err != nil { - return errors.Wrapf(err, "failed to create service %s in namespace %s", service.Name, service.Namespace) - } - } else { - return errors.Wrapf(err, "failed to update service %s in namespace %s", service.Name, service.Namespace) - } - } - } +func (i *PropagationController) removePropagation(ctx context.Context, prop propagation.Propagation) error { + err := i.TargetClient.Delete(ctx, &prop.Ingress) + if !k8serrors.IsNotFound(err) { + if err != nil { + return fmt.Errorf("failed to delete ingress %s: %s", prop.Ingress.Name, err) } } + + i.Recorder.Eventf(&prop.Origin, corev1.EventTypeNormal, "IngressUnpropagated", "Ingress has been removed") return nil } diff --git a/pkg/controller/transform.go b/pkg/controller/transform.go index 3e0c687..c1fff0f 100644 --- a/pkg/controller/transform.go +++ b/pkg/controller/transform.go @@ -9,95 +9,124 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/go-logr/logr" - "github.com/pkg/errors" v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) -func FromIngressToPropagation(ctx context.Context, logger logr.Logger, kubeClient client.Client, ingress networkingv1.Ingress, ingressClass string, identifier string, namespace string) (propagation.Propagation, error) { - result := propagation.Propagation{} - ing := networkingv1.Ingress{} - result.IsDeleted = false +func (i *PropagationController) FromIngressToPropagation(ctx context.Context, logger logr.Logger, kubeClient client.Client, ingress networkingv1.Ingress) (propagation.Propagation, error) { + result := propagation.Propagation{ + Name: ingress.Name, + PropagatedName: fmt.Sprintf("%s-%s", i.Options.Identifier, ingress.Name), + IsDeleted: false, + Ingress: networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", i.Options.Identifier, ingress.Name), + Namespace: i.Options.TargetNamespace, + }, + }, + Origin: ingress, + } if ingress.DeletionTimestamp != nil { result.IsDeleted = true - } + } else { - // Assign Name - result.Name = ingress.Name - ingress.SetNamespace(namespace) + // Assign Labels + result.Ingress.Labels = make(map[string]string) + if ingress.Labels != nil { + result.Ingress.Labels = ingress.Labels + } + result.Ingress.Labels[LabelManaged] = i.Options.Identifier - if ingress.Spec.TLS != nil { - ing.Spec.TLS = ingress.Spec.TLS - } + // Annotations + result.Ingress.Annotations = make(map[string]string) + if ingress.Annotations != nil { + result.Ingress.Annotations = ingress.Annotations + } - // Assign Labels - if ingress.Labels == nil { - ingress.Labels = make(map[string]string) - } - ingress.Labels[LabelManaged] = identifier + result.Ingress.Spec.IngressClassName = &i.Options.TargetIngressClassName - ingress.Spec.IngressClassName = &ingressClass + // Store relevant Services + var hosts []string + var services []v1.Service - // Store relevant Services - var services []v1.Service + for r := range ingress.Spec.Rules { + rule := &ingress.Spec.Rules[r] + if rule.Host == "" { + return result, fmt.Errorf("host in ingress %s/%s is empty", ingress.GetNamespace(), ingress.GetName()) + } - for _, rule := range ingress.Spec.Rules { - if rule.Host == "" { - return result, errors.Errorf("host in ingress %s/%s is empty", ingress.GetNamespace(), ingress.GetName()) - } + for p := range rule.HTTP.Paths { + path := &rule.HTTP.Paths[p] - for _, path := range rule.HTTP.Paths { + namespacedName := types.NamespacedName{ + Namespace: ingress.GetNamespace(), + Name: path.Backend.Service.Name, + } + service := v1.Service{} + err := kubeClient.Get(ctx, namespacedName, &service) + if err != nil { + return result, fmt.Errorf("fetch service %s: %s", namespacedName, err) + } - namespacedName := types.NamespacedName{ - Namespace: ingress.GetNamespace(), - Name: path.Backend.Service.Name, - } - service := v1.Service{} - err := kubeClient.Get(ctx, namespacedName, &service) - if err != nil { - return result, errors.Wrapf(err, "fetch service %s", namespacedName) - } + if service.Status.LoadBalancer.Ingress == nil { + return result, fmt.Errorf("service %s has no loadbalancer ip", namespacedName) + } - if service.Status.LoadBalancer.Ingress == nil { - return result, errors.Errorf("service %s has no loadbalancer ip", namespacedName) - } + var port int32 + if path.Backend.Service.Port.Name != "" { + ok, extractedPort := getPortWithName(service.Spec.Ports, path.Backend.Service.Port.Name) + if !ok { + return result, fmt.Errorf("service %s has no port named %s", namespacedName, path.Backend.Service.Port.Name) + } + port = extractedPort + } else { + port = path.Backend.Service.Port.Number + } - if !containsService(services, path.Backend.Service.Name) { - services = append(services, service) - } + service.ObjectMeta.Name = result.PropagatedName + if !containsService(services, path.Backend.Service.Name) { + services = append(services, service) + } - var port int32 - if path.Backend.Service.Port.Name != "" { - ok, extractedPort := getPortWithName(service.Spec.Ports, path.Backend.Service.Port.Name) - if !ok { - return result, errors.Errorf("service %s has no port named %s", namespacedName, path.Backend.Service.Port.Name) + path.Backend = networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: result.PropagatedName, + Port: networkingv1.ServiceBackendPort{ + Number: port, + }, + }, } - port = extractedPort - } else { - port = path.Backend.Service.Port.Number } + result.Ingress.Spec.Rules = append(result.Ingress.Spec.Rules, *rule) + hosts = append(hosts, rule.Host) + } - path.Backend = networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: result.Name, - Port: networkingv1.ServiceBackendPort{ - Number: port, - }, - }, + // Add TLS information + if (i.Options.TLSrespect) && (ingress.Spec.TLS != nil) { + result.Ingress.Spec.TLS = ingress.Spec.TLS + } + + if i.Options.TargetIssuerName != "" { + if i.Options.TargetIssuerNamespaced { + result.Ingress.Annotations[IssuerNamespacedAnnotation] = i.Options.TargetIssuerName + } else { + result.Ingress.Annotations[IssuerClusterAnnotation] = i.Options.TargetIssuerName } + result.Ingress.Spec.TLS = append(result.Ingress.Spec.TLS, networkingv1.IngressTLS{ + Hosts: hosts, + SecretName: result.PropagatedName, + }) } - } - result.Ingress = ingress - // Load Services and endpoints - error := resolveServiceEndpoints(services, &result, identifier, namespace) - if error != nil { - return result, errors.Wrapf(error, "failed to resolve service endpoints") + // Load Services and endpoints + err := resolveServiceEndpoints(services, &result, i.Options.Identifier, i.Options.TargetNamespace) + if err != nil { + return result, fmt.Errorf("failed to resolve service endpoints: %s", err) + } } - return result, nil } @@ -164,7 +193,7 @@ func resolveServiceEndpoints(services []v1.Service, propagation *propagation.Pro } propagation.Endpoints = append(propagation.Endpoints, endpoint) } - fmt.Printf("HIHO %v\n\n", propagation) + return nil } diff --git a/pkg/controller/utils.go b/pkg/controller/utils.go index cd11529..29e8a76 100644 --- a/pkg/controller/utils.go +++ b/pkg/controller/utils.go @@ -1,14 +1,13 @@ package controller -func removeStringFromSlice(finalizers []string, finalizer string) []string { - var result []string - for _, f := range finalizers { - if f != finalizer { - result = append(result, f) - } - } - return result -} +const WellKnownIngressAnnotation = "kubernetes.io/ingress.class" +const MetaBase = "ingress-propagator.buttah.cloud" + +var LabelManaged = MetaBase + "/managed-by" +var LabelPropagator = MetaBase + "/propagator" + +const IssuerNamespacedAnnotation = "cert-manager.io/issuer" +const IssuerClusterAnnotation = "cert-manager.io/cluster-issuer" func stringSliceContains(slice []string, element string) bool { for _, sliceElement := range slice { diff --git a/pkg/propagation/propagation.go b/pkg/propagation/propagation.go index d0ee170..8b0889d 100644 --- a/pkg/propagation/propagation.go +++ b/pkg/propagation/propagation.go @@ -10,9 +10,14 @@ type Propagation struct { // Propagation Name Name string + PropagatedName string + // State if the propagation is deleted on the child cluster IsDeleted bool + // Origin Ingress + Origin networkingv1.Ingress + // The ingress object associated with the propagation. Ingress networkingv1.Ingress