diff --git a/.cruft.json b/.cruft.json index cb977e8..c98f6e6 100644 --- a/.cruft.json +++ b/.cruft.json @@ -7,7 +7,7 @@ "name": "openshift4-config", "slug": "openshift4-config", "parameter_key": "openshift4_config", - "test_cases": "defaults", + "test_cases": "defaults pull-secret", "add_lib": "n", "add_pp": "n", "add_golden": "y", diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0b8d045..5ad04cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,7 @@ jobs: matrix: instance: - defaults + - pull-secret defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -48,6 +49,7 @@ jobs: matrix: instance: - defaults + - pull-secret defaults: run: working-directory: ${{ env.COMPONENT_NAME }} diff --git a/Makefile.vars.mk b/Makefile.vars.mk index c8c5acf..98ed72e 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -50,4 +50,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults -test_instances = tests/defaults.yml +test_instances = tests/defaults.yml tests/pull-secret.yml diff --git a/class/defaults.yml b/class/defaults.yml index 0eb52fd..d6e3dcd 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -1,7 +1,13 @@ parameters: openshift4_config: =_metadata: {} - globalPullSecret: null + globalPullSecrets: {} + + images: + kubectl: + registry: docker.io + repository: bitnami/kubectl + tag: 1.25.4 # Fixes cluster upgrades on OCP4.10 clusters with custom `privileged` SCCs. clusterUpgradeSCCPermissionFix: diff --git a/component/main.jsonnet b/component/main.jsonnet index 6360aff..ed76645 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -5,18 +5,31 @@ local inv = kap.inventory(); // The hiera parameters for the component local params = inv.parameters.openshift4_config; -local dockercfg = kube.Secret('pull-secret') { - metadata+: { - namespace: 'openshift-config', - }, - stringData+: { - '.dockerconfigjson': params.globalPullSecret, - }, - type: 'kubernetes.io/dockerconfigjson', -}; +local legacyPullSecret = std.get(params, 'globalPullSecret', null); + +local dockercfg = std.trace( + 'Your config for openshift4-config uses the deprecated `globalPullSecret` parameter. ' + + 'Please migrate to `globalPullSecrets`. ' + + 'See https://hub.syn.tools/openshift4-config/how-to/migrate-v1.html for details.', + kube.Secret('pull-secret') { + metadata+: { + namespace: 'openshift-config', + annotations+: { + 'argocd.argoproj.io/sync-options': 'Prune=false', + }, + }, + stringData+: { + '.dockerconfigjson': legacyPullSecret, + }, + type: 'kubernetes.io/dockerconfigjson', + } +); // Define outputs below { - [if params.globalPullSecret != null then '01_dockercfg']: dockercfg, - [if params.clusterUpgradeSCCPermissionFix.enabled then '02_clusterUpgradeSCCPermissionFix']: (import 'privileged-scc.libsonnet'), + [if legacyPullSecret != null then '01_dockercfg']: dockercfg, + [if legacyPullSecret == null && std.length(std.objectFields(params.globalPullSecrets)) > 0 then '99_cluster_pull_secret']: + import 'pull-secret-sync-job.libsonnet', + [if params.clusterUpgradeSCCPermissionFix.enabled then '02_clusterUpgradeSCCPermissionFix']: + import 'privileged-scc.libsonnet', } diff --git a/component/pull-secret-sync-job.libsonnet b/component/pull-secret-sync-job.libsonnet new file mode 100644 index 0000000..c3a57ff --- /dev/null +++ b/component/pull-secret-sync-job.libsonnet @@ -0,0 +1,167 @@ +// Template for the ArgoCD sync job to manage the OpenShift cluster pull +// secret. +// The job is modelled after the instructions outlined in +// https://docs.openshift.com/container-platform/4.11/post_installation_configuration/cluster-tasks.html#images-update-global-pull-secret_post-install-cluster-tasks +local kap = import 'lib/kapitan.libjsonnet'; +local kube = import 'lib/kube.libjsonnet'; +local inv = kap.inventory(); +// The hiera parameters for the component +local params = inv.parameters.openshift4_config; + +// Jobs need get,update,patch for secret pull-secret in namespace openshift-config +// To ensure the unmanage job has the RBAC in place, all the RBAC objects are +// also in sync-wave -10. +local jobSA = kube.ServiceAccount('syn-cluster-pull-secret-manager') { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-wave': '-10', + }, + namespace: 'openshift-config', + }, +}; +local jobRole = kube.Role('syn-cluster-pull-secret-manager') { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-wave': '-10', + }, + }, + rules: [ + { + apiGroups: [ '' ], + resources: [ 'secrets' ], + verbs: [ 'get', 'update', 'patch' ], + resourceNames: [ 'pull-secret' ], + }, + ], +}; +local jobRoleBinding = kube.RoleBinding('syn-cluster-pull-secret-manager') { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-wave': '-10', + }, + }, + roleRef_: jobRole, + subjects_: [ jobSA ], +}; + +local cleanJob = kube.Job('syn-unmanage-cluster-pull-secret') { + metadata+: { + annotations+: { + // run before the default sync wave, but after creating the Job RBAC so + // that we unmanage the cluster pull secret before patching it. + 'argocd.argoproj.io/sync-wave': '-9', + 'argocd.argoproj.io/hook': 'Sync', + 'argocd.argoproj.io/hook-delete-policy': 'HookSucceeded', + }, + }, + spec+: { + template+: { + spec+: { + serviceAccountName: jobSA.metadata.name, + containers_: { + clean: { + image: '%(registry)s/%(repository)s:%(tag)s' % params.images.kubectl, + command: [ + 'bash', + '-ce', + 'kubectl label secret pull-secret argocd.argoproj.io/instance-;' + + 'kubectl annotate secret pull-secret kubectl.kubernetes.io/last-applied-configuration-;' + + 'kubectl annotate secret pull-secret argocd.argoproj.io/sync-options-;', + ], + }, + }, + }, + }, + }, +}; + +local syncScript = kube.Secret('syn-update-cluster-pull-secret-script') { + stringData: { + // The shell script reads the secret `pull-secret`, base64-decodes the + // value of `.dockerconfigjson`, processes it with jq and updates the + // secret with the result of the JQ script (see below). + 'sync-secret.sh': ||| + #!/bin/bash + set -eu + + pull_secret=$( + kubectl get secret pull-secret \ + -o go-template='{{index .data ".dockerconfigjson"|base64decode}}' + ) + patched_secret=$( + jq -cr '%(script)s' <<<"${pull_secret}" + ) + kubectl -n openshift-config patch secret pull-secret \ + -p "{\"data\": {\".dockerconfigjson\": \"$patched_secret\"}}" + ||| % { + // We generate a JQ script which processes the pull-secret contents from + // params.globalPullSecrets. For each entry in the parameter, we + // generate a `.auths.[key]=[value]`. Jsonnet string formatting + // automatically formats objects as valid JSON when formatting them with + // %s. After processing each entry of the parameter, the script runs + // `del(..|nulls)` to drop any keys with `null` values and `@base64` to + // base64-encode the resulting object. + script: + // We transform the globalPullSecrets object into a list of objects + // representing the entries of the object... + local pullSecretKV = [ + { + key: k, + value: params.globalPullSecrets[k], + } + for k in std.objectFields(params.globalPullSecrets) + ]; + // We use the transformed parameter to generate `.auths."[key]"=value` + // for each entry... + local auth_patches = std.foldl(function(str, cfg) str + '.auths."%(key)s"=%(value)s |' % cfg, pullSecretKV, ''); + // and finally we append `del(..|nulls)|@base64` to the script. + auth_patches + 'del(..|nulls)|@base64', + }, + }, +}; + +local syncJob = kube.Job('syn-update-cluster-pull-secret') { + metadata+: { + annotations+: { + // run after the default sync wave since we depend on the script secret. + 'argocd.argoproj.io/sync-wave': '10', + 'argocd.argoproj.io/hook': 'Sync', + 'argocd.argoproj.io/hook-delete-policy': 'HookSucceeded', + }, + }, + spec+: { + template+: { + spec+: { + serviceAccountName: jobSA.metadata.name, + containers_: { + update: kube.Container('update') { + image: '%(registry)s/%(repository)s:%(tag)s' % params.images.kubectl, + command: [ '/script/sync-secret.sh' ], + volumeMounts_: { + script: { + mountPath: '/script', + }, + }, + }, + }, + volumes_: { + script: { + secret: { + secretName: syncScript.metadata.name, + defaultMode: 504, // 0770 + }, + }, + }, + }, + }, + }, +}; + +[ + jobSA, + jobRole, + jobRoleBinding, + cleanJob, + syncScript, + syncJob, +] diff --git a/docs/modules/ROOT/pages/how-tos/migrate-v1.adoc b/docs/modules/ROOT/pages/how-tos/migrate-v1.adoc new file mode 100644 index 0000000..fdf184b --- /dev/null +++ b/docs/modules/ROOT/pages/how-tos/migrate-v1.adoc @@ -0,0 +1,33 @@ += Migrate from v0.x to v1.x + +== Migrate cluster pull secret config + +If your cluster config uses `globalPullSecret` parameter, you should migrate your customizations to parameter `globalPullSecrets`. + +If you've added additional pull secrets, you can now configure them as + +[source,yaml] +---- +parameters: + openshift4_config: + globalPullSecrets: + registry.example.com: + auth: ?{vaultkv:${cluster:tenant}/${cluster:name}/openshift4-config/registry.example.com-pull-secret} <1> + email: docker@example.com <2> +---- +<1> We strongly recommend that you store the `auth` config for the additional registry in Vault. +Please make sure you store the config as a base64-encoded string in Vault. +<2> Some registries require an email address for authenticated pulls. + +If you've removed pull secrets, for example to https://docs.openshift.com/container-platform/4.11/support/remote_health_monitoring/opting-out-of-remote-health-reporting.html#insights-operator-new-pull-secret_opting-out-remote-health-reporting[disable telemetry], you can now remove them with + +[source,yaml] +---- +parameters: + openshift4_config: + globalPullSecrets: + cloud.openshift.com: null <1> +---- +<1> Setting a registry hostname to `null` will remove any auth config for that registry if it's present in the `pull-secret` secret on the cluster. + +See the xref:references/parameters.adoc#_globalPullSecrets[parameter docs] for more details. diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index f0e948b..7273d3e 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -6,7 +6,13 @@ The parent key for all the following parameters is `openshift4_config`. [horizontal] type:: string -default:: null +default:: absent + +[IMPORTANT] +==== +This parameter is deprecated. +Please migrate your additional pull secrets to parameter `globalPullSecrets`. +==== A Vault reference pointing to the Vault secret containing the docker configuration file in JSON format. If the parameter is null, the component doesn't manage the cluster's global pull secret. @@ -21,6 +27,32 @@ You need to make sure that the existing pull secrets present on a cluster (deplo Otherwise, OpenShift cluster services may stop working because their respective container images can't be downloaded anymore. ==== +== `globalPullSecrets` + +[horizontal] +type:: dict +default:: `{}` +example:: ++ +[source,yaml] +---- +docker.io: + email: dockerhub@example.com <1> + auth: ?{vaultkv:${cluster:tenant}/${cluster:name}/openshift4-config/docker.io-pull-secret} <2> +---- +<1> Some registries require an email address to be present for authenticated pulls. +<2> We strongly recommend that you store the `auth` value for the registry in Vault. + +This parameter allows customizing the OpenShift cluster pull-secret without having to replicate the complete secret contents in Vault. +The component expects entries in the dict to be valid entries for the `.dockerconfigjson` `auths` field. +The component allows users to remove existing entries (also entries originally created by the OpenShift installer) by setting the value for a registry host to `null`. + +[NOTE] +==== +The component doesn't base64-encode the value provided for `auth`. +Please make sure that you store the `auth` value as base64 in Vault. +==== + == `clusterUpgradeSCCPermissionFix` [horizontal] diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc index 08f9283..02136ca 100644 --- a/docs/modules/ROOT/partials/nav.adoc +++ b/docs/modules/ROOT/partials/nav.adoc @@ -1,2 +1,7 @@ * xref:index.adoc[Home] + +.How-to guides +* xref:how-tos/migrate-v1.adoc[] + +.Technical reference * xref:references/parameters.adoc[Parameters] diff --git a/tests/defaults.yml b/tests/defaults.yml index f359744..51fa235 100644 --- a/tests/defaults.yml +++ b/tests/defaults.yml @@ -1,3 +1,2 @@ parameters: - openshift4_config: - globalPullSecret: ?{vaultkv:${cluster:tenant}/${cluster:name}/openshift4-config/dockercfg} + openshift4_config: {} diff --git a/tests/golden/defaults/openshift4-config/openshift4-config/01_dockercfg.yaml b/tests/golden/defaults/openshift4-config/openshift4-config/01_dockercfg.yaml deleted file mode 100644 index eb0b515..0000000 --- a/tests/golden/defaults/openshift4-config/openshift4-config/01_dockercfg.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -data: {} -kind: Secret -metadata: - annotations: {} - labels: - name: pull-secret - name: pull-secret - namespace: openshift-config -stringData: - .dockerconfigjson: t-silent-test-1234/c-green-test-1234/openshift4-config/dockercfg -type: kubernetes.io/dockerconfigjson diff --git a/tests/golden/pull-secret/openshift4-config/apps/openshift4-config.yaml b/tests/golden/pull-secret/openshift4-config/apps/openshift4-config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/pull-secret/openshift4-config/openshift4-config/02_clusterUpgradeSCCPermissionFix.yaml b/tests/golden/pull-secret/openshift4-config/openshift4-config/02_clusterUpgradeSCCPermissionFix.yaml new file mode 100644 index 0000000..907aecb --- /dev/null +++ b/tests/golden/pull-secret/openshift4-config/openshift4-config/02_clusterUpgradeSCCPermissionFix.yaml @@ -0,0 +1,54 @@ +allowHostDirVolumePlugin: true +allowHostIPC: true +allowHostNetwork: true +allowHostPID: true +allowHostPorts: true +allowPrivilegeEscalation: true +allowPrivilegedContainer: true +allowedCapabilities: + - '*' +allowedUnsafeSysctls: + - '*' +apiVersion: security.openshift.io/v1 +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +groups: + - system:cluster-admins + - system:nodes + - system:masters +kind: SecurityContextConstraints +metadata: + annotations: + kubernetes.io/description: 'Copy of `privileged` with increased priority to be + choosen over other custom SCCs. + + + privileged allows access to all privileged and host features and the ability + to run as any user, any group, any fsGroup, and with any SELinux context. + + WARNING: this is the most relaxed SCC and should be used only for cluster administration. + Grant with caution. + + ' + labels: + app.kubernetes.io/component: openshift4-config + app.kubernetes.io/managed-by: commodore + name: privileged-higher-prio + name: privileged-higher-prio +priority: 3 +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: + - '*' +supplementalGroups: + type: RunAsAny +users: + - system:admin + - system:serviceaccount:openshift-infra:build-controller +volumes: + - '*' diff --git a/tests/golden/pull-secret/openshift4-config/openshift4-config/99_cluster_pull_secret.yaml b/tests/golden/pull-secret/openshift4-config/openshift4-config/99_cluster_pull_secret.yaml new file mode 100644 index 0000000..fb8a958 --- /dev/null +++ b/tests/golden/pull-secret/openshift4-config/openshift4-config/99_cluster_pull_secret.yaml @@ -0,0 +1,141 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + argocd.argoproj.io/sync-wave: '-10' + labels: + name: syn-cluster-pull-secret-manager + name: syn-cluster-pull-secret-manager + namespace: openshift-config +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + argocd.argoproj.io/sync-wave: '-10' + labels: + name: syn-cluster-pull-secret-manager + name: syn-cluster-pull-secret-manager +rules: + - apiGroups: + - '' + resourceNames: + - pull-secret + resources: + - secrets + verbs: + - get + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + argocd.argoproj.io/sync-wave: '-10' + labels: + name: syn-cluster-pull-secret-manager + name: syn-cluster-pull-secret-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: syn-cluster-pull-secret-manager +subjects: + - kind: ServiceAccount + name: syn-cluster-pull-secret-manager + namespace: openshift-config +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + argocd.argoproj.io/sync-wave: '-9' + labels: + name: syn-unmanage-cluster-pull-secret + name: syn-unmanage-cluster-pull-secret +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + name: syn-unmanage-cluster-pull-secret + spec: + containers: + - command: + - bash + - -ce + - kubectl label secret pull-secret argocd.argoproj.io/instance-;kubectl + annotate secret pull-secret kubectl.kubernetes.io/last-applied-configuration-;kubectl + annotate secret pull-secret argocd.argoproj.io/sync-options-; + image: docker.io/bitnami/kubectl:1.25.4 + name: clean + imagePullSecrets: [] + initContainers: [] + restartPolicy: OnFailure + serviceAccountName: syn-cluster-pull-secret-manager + terminationGracePeriodSeconds: 30 + volumes: [] +--- +apiVersion: v1 +data: {} +kind: Secret +metadata: + annotations: {} + labels: + name: syn-update-cluster-pull-secret-script + name: syn-update-cluster-pull-secret-script +stringData: + sync-secret.sh: "#!/bin/bash\nset -eu\n\npull_secret=$(\n kubectl get secret\ + \ pull-secret \\\n -o go-template='{{index .data \".dockerconfigjson\"|base64decode}}'\n\ + )\npatched_secret=$(\n jq -cr '.auths.\"ghcr.io\"={\"auth\": \"t-silent-test-1234/c-green-test-1234/openshift4-config/pullSecrets/ghcr.io-pull-secret\"\ + , \"email\": \"github@example.com\"} |.auths.\"quay.io\"=null |del(..|nulls)|@base64'\ + \ <<<\"${pull_secret}\"\n)\nkubectl -n openshift-config patch secret pull-secret\ + \ \\\n -p \"{\\\"data\\\": {\\\".dockerconfigjson\\\": \\\"$patched_secret\\\"\ + }}\"\n" +type: Opaque +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + argocd.argoproj.io/sync-wave: '10' + labels: + name: syn-update-cluster-pull-secret + name: syn-update-cluster-pull-secret +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + name: syn-update-cluster-pull-secret + spec: + containers: + - args: [] + command: + - /script/sync-secret.sh + env: [] + image: docker.io/bitnami/kubectl:1.25.4 + imagePullPolicy: IfNotPresent + name: update + ports: [] + stdin: false + tty: false + volumeMounts: + - mountPath: /script + name: script + imagePullSecrets: [] + initContainers: [] + restartPolicy: OnFailure + serviceAccountName: syn-cluster-pull-secret-manager + terminationGracePeriodSeconds: 30 + volumes: + - name: script + secret: + defaultMode: 504 + secretName: syn-update-cluster-pull-secret-script diff --git a/tests/pull-secret.yml b/tests/pull-secret.yml new file mode 100644 index 0000000..7c62b30 --- /dev/null +++ b/tests/pull-secret.yml @@ -0,0 +1,9 @@ +# Overwrite parameters here + +parameters: + openshift4_config: + globalPullSecrets: + ghcr.io: + email: github@example.com + auth: ?{vaultkv:${cluster:tenant}/${cluster:name}/openshift4-config/pullSecrets/ghcr.io-pull-secret} + quay.io: null