feat(seeding-job)!: enable realm import with dynamic config (#141)

- enable seeding job for realm import and upgrade with dynamic configuration of redirect urls, client secrets, etc. for centralidp and sharedidp - previously only used for upgrading the CX-Central realm configuration for centralidp
- improve secret handling and remove obsolete secrets
- move to standalone architecture for database dependency
- set default replica count to 1
- update chart testing

.github/workflows/centralidp-chart-test.yaml

@@ -103,7 +103,7 @@ jobs:
         run: |
           helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
           helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev
-          helm install centralidp tractusx-dev/centralidp --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
+          helm install centralidp tractusx-dev/centralidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
           helm dependency update charts/centralidp
-          helm upgrade centralidp charts/centralidp -f charts/values-test-centralidp.yaml --namespace upgrade --debug
+          helm upgrade centralidp charts/centralidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug
         if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

.github/workflows/sharedidp-chart-test.yaml

@@ -103,7 +103,7 @@ jobs:
         run: |
           helm repo add bitnami-full-index https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
           helm repo add tractusx-dev https://eclipse-tractusx.github.io/charts/dev
-          helm install sharedidp tractusx-dev/sharedidp --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
+          helm install sharedidp tractusx-dev/sharedidp -f charts/values-test-upgrade.yaml --version ${{ github.event.inputs.upgrade_from || '2.1.0' }} --namespace upgrade --create-namespace --debug
           helm dependency update charts/sharedidp
-          helm upgrade sharedidp charts/sharedidp -f charts/values-test-sharedidp.yaml --namespace upgrade --debug
+          helm upgrade sharedidp charts/sharedidp -f charts/values-test-upgrade.yaml --namespace upgrade --debug
         if: github.event_name != 'pull_request' || steps.list-changed.outputs.changed == 'true'

README.md

@@ -17,12 +17,10 @@ For further information please refer to the chart specific README files, availab
 
 * charts/centralidp
   * [Installation](./charts/centralidp/README.md#installation)
-  * [Post-Install Configuration](./charts/centralidp/README.md#post-install-configuration)
   * [Upgrade](./charts/centralidp/README.md#upgrade)
   * [Post-Upgrade Configuration](./charts/centralidp/README.md#post-upgrade-configuration)
 * charts/sharedidp
   * [Installation](./charts/sharedidp/README.md#installation)
-  * [Post-Install Configuration](./charts/sharedidp/README.md#post-install-configuration)
   * [Upgrade](./charts/sharedidp/README.md#upgrade)
 
 ## Known Issues and Limitations

charts/centralidp/README.md.gotmpl

@@ -38,19 +38,6 @@ dependencies:
 
 Autogenerated with [helm docs](https://github.com/norwoodj/helm-docs)
 
-## Post-Install Configuration
-
-Once the installation is completed, the following steps need to be executed in the Keycloak admin console within the CX-Central realm:
-
-1. Generate client-secrets for confidential clients and service accounts with access type 'confidential'.
-
-2. Establish connection to the sharedidp instance
-
-In order to enable the login of the initial user (see CX-Operator realm in sharedidp instance for username), the connection between the 'CX-Operator' identity provider of the centralidp instance and the according realm in the sharedidp instance needs to be established.
-This is done by setting the 'example.org' placeholder in the CX-Operator' Identity Provider to the address of the sharedidp instance.
-
-3. Setup SMTP configuration (Realm Settings --> Email)
-
 ## Upgrade
 
 Please see notes at [Values.seeding](values.yaml#L153) for upgrading the configuration of the CX-Central realm.

charts/centralidp/templates/_helpers.tpl

@@ -15,7 +15,9 @@
 * under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
+*/}}
 
+{{/*
 Define "centralidp.fullname" like ""common.names.fullname" in the bitnami common chart but setting ".Chart.Name" to "keycloak".
 This is necessary to retrieve the keycloak service name for the execution of the seeding job.
 */}}
@@ -31,3 +33,36 @@ This is necessary to retrieve the keycloak service name for the execution of the
 {{- end -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Define secret name for clients secrets of clients (realm seeding).
+*/}}
+{{- define "centralidp.secret.clients" -}}
+{{- if .Values.realmSeeding.clients.existingSecret -}}
+{{- .Values.realmSeeding.clients.existingSecret }}
+{{- else -}}
+{{- include "centralidp.fullname" . -}}-clients
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define secret name for clients secrets of base service accounts (realm seeding).
+*/}}
+{{- define "centralidp.secret.serviceAccounts" -}}
+{{- if .Values.realmSeeding.serviceAccounts.existingSecret -}}
+{{- .Values.realmSeeding.serviceAccounts.existingSecret }}
+{{- else -}}
+{{- include "centralidp.fullname" . -}}-base-service-accounts
+{{- end -}}
+{{- end -}}
+
+{{/*
+Define secret name for clients secrets of additional service accounts (realm seeding).
+*/}}
+{{- define "centralidp.secret.extraServiceAccounts" -}}
+{{- if .Values.realmSeeding.extraServiceAccounts.existingSecret -}}
+{{- .Values.realmSeeding.extraServiceAccounts.existingSecret }}
+{{- else -}}
+{{- include "centralidp.fullname" . -}}-extra-service-accounts
+{{- end -}}
+{{- end -}}

charts/centralidp/templates/job-seeding.yaml

@@ -17,64 +17,235 @@
 * SPDX-License-Identifier: Apache-2.0
 */}}
 
-{{- if .Values.seeding.enabled -}}
+{{- if .Values.realmSeeding.enabled -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "centralidp.fullname" . }}-upgrade-seeding
+  name: {{ include "centralidp.fullname" . }}-realm-seeding-{{ .Release.Revision }}
   namespace: {{ .Release.Namespace }}
-  annotations:
-    "batch.kubernetes.io/job-tracking": "true"
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-5"
 spec:
   template:
     metadata:
-      name: {{ include "centralidp.fullname" . }}-upgrade-seeding
+      name: {{ include "centralidp.fullname" . }}-realm-seeding
     spec:
       restartPolicy: Never
       containers:
-      - name: {{ include "centralidp.fullname" . }}-upgrade-seeding
+      - name: {{ include "centralidp.fullname" . }}-realm-seeding
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
             - ALL
+          readOnlyRootFilesystem: true
           runAsNonRoot: true
-        image: "{{ .Values.seeding.image }}"
-        imagePullPolicy: "{{ .Values.seeding.imagePullPolicy }}"
+        image: "{{ .Values.realmSeeding.image.name }}"
+        imagePullPolicy: "{{ .Values.realmSeeding.image.pullPolicy }}"
         env:
           - name: "KEYCLOAK__CENTRAL__CONNECTIONSTRING"
-            value: "http://{{ template "centralidp.fullname" . }}"
+            value: |
+              {{ if .Values.realmSeeding.keycloakServiceTls -}}
+              https://{{ template "centralidp.fullname" . }}:{{ .Values.realmSeeding.keycloakServicePort }}
+              {{- else -}}
+              http://{{ template "centralidp.fullname" . }}:{{ .Values.realmSeeding.keycloakServicePort }}
+              {{- end }}
           - name: "KEYCLOAK__CENTRAL__USER"
             value: "{{ .Values.keycloak.auth.adminUser }}"
           - name: "KEYCLOAK__CENTRAL__PASSWORD"
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.keycloak.auth.existingSecret }}"
+                name: "{{ template "centralidp.fullname" . }}"
                 key: "admin-password"
           - name: "KEYCLOAK__CENTRAL__AUTHREALM"
-            value: "{{ .Values.seeding.authRealm }}"
+            value: "master"
           - name: "KEYCLOAK__CENTRAL__USEAUTHTRAIL"
-            value: "{{ .Values.seeding.useAuthTrail }}"
-          - name: "KEYCLOAKSEEDING__DATAPATHES__0"
-            value: "{{ .Values.seeding.dataPaths.dataPath0 }}"
-          - name: "KEYCLOAKSEEDING__INSTANCENAME"
-            value: "{{ .Values.seeding.instanceName }}"
-          - name: "KEYCLOAKSEEDING__EXCLUDEDUSERATTRIBUTES__0"
-            value: "{{ .Values.seeding.excludedUserAttributes.attribute0 }}"
-          - name: "KEYCLOAKSEEDING__EXCLUDEDUSERATTRIBUTES__1"
-            value: "{{ .Values.seeding.excludedUserAttributes.attribute1 }}"
+            value: "true"
+          - name: "KEYCLOAKSEEDING__REALMS__0__DATAPATHS__0"
+            value: "realms/CX-Central-realm.json"
+          - name: "KEYCLOAKSEEDING__REALMS__0__INSTANCENAME"
+            value: "central"
+          - name: "KEYCLOAKSEEDING__REALMS__0__REALM"
+            value: "CX-Central"
+
+          #############################
+          ## INITIAL USER
+          #############################
+
+          - name: "KEYCLOAKSEEDING__REALMS__0__DATAPATHS__1"
+            value: "realms/CX-Central-users-0.json"
+
+          #############################
+          ## CLIENTS
+          #############################
+
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__0__CLIENTID"
+            value: "Cl1-CX-Registration"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.registration.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__0__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__1__CLIENTID"
+            value: "Cl2-CX-Portal"
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__1__ROOTURL"
+            value: "{{ .Values.realmSeeding.clients.portal.rootUrl }}"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.portal.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__1__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__2__CLIENTID"
+            value: "Cl3-CX-Semantic"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.semantics.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__2__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__3__CLIENTID"
+            value: "Cl5-CX-Custodian"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.miw.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__3__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__3__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ template "centralidp.secret.clients" . }}"
+                key: "miw"
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__4__CLIENTID"
+            value: "Cl7-CX-BPDM"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.bpdm.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__4__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__4__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ template "centralidp.secret.clients" . }}"
+                key: "bpdm"
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__5__CLIENTID"
+            value: "Cl16-CX-BPDMGate"
+          {{- range $index, $redirects := .Values.realmSeeding.clients.bpdmGate.redirects }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__5__REDIRECTURIS__{{ $index }}"
+            value: {{ $redirects | quote }}
+          {{- end }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__5__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ template "centralidp.secret.clients" . }}"
+                key: "bpdm-gate"
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__6__CLIENTID"
+            value: "Cl25-CX-BPDM-Orchestrator"
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__6__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ template "centralidp.secret.clients" . }}"
+                key: "bpdm-orchestrator"
+
+          #############################
+          ## BASE SERVICE ACCOUNTS
+          #############################
+
+          {{- $index := 20 }}
+          {{- $secretBaseSa := include "centralidp.secret.serviceAccounts" . -}}
+          {{- range .Values.realmSeeding.serviceAccounts.clientSecrets }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__{{ $index }}__CLIENTID"
+            value: {{ .clientId | quote }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__{{ $index }}__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ $secretBaseSa }}"
+                key: "{{ .clientId }}"
+          {{- $index = add1 $index }}
+          {{- end }}
+
+          #############################
+          ## BPN USER ATTRIBUTE
+          #############################
+
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__0__USERNAME"
+            value: "ac1cf001-7fbc-1f2f-817f-bce058020006"
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__0__Attributes__0__NAME"
+            value: "bpn"
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__0__Attributes__0__VALUES__0"
+            value: "{{ .Values.realmSeeding.bpn }}"
+          {{- $index := 1 }}
+          {{- $bpn := .Values.realmSeeding.bpn -}}
+          {{- range .Values.realmSeeding.serviceAccounts.clientSecrets }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__{{ $index }}__USERNAME"
+            value: {{ printf "service-account-%s" .clientId | quote }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__{{ $index }}__ATTRIBUTES__0__NAME"
+            value: "bpn"
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__{{ $index }}__ATTRIBUTES__0__VALUES__0"
+            value: {{ $bpn | quote }}
+          {{- $index = add1 $index }}
+          {{- end }}
+
+          #############################
+          ## IDENTITY PROVIDER
+          #############################
+
+          - name: "KEYCLOAKSEEDING__REALMS__0__IDENTITYPROVIDERS__0__ALIAS"
+            value: "CX-Operator"
+          - name: "KEYCLOAKSEEDING__REALMS__0__IDENTITYPROVIDERS__0__CONFIG__TOKENURL"
+            value: "{{ .Values.realmSeeding.sharedidp }}/auth/realms/CX-Operator/protocol/openid-connect/token"
+          - name: "KEYCLOAKSEEDING__REALMS__0__IDENTITYPROVIDERS__0__CONFIG__JWKSURL"
+            value: "{{ .Values.realmSeeding.sharedidp }}/auth/realms/CX-Operator/protocol/openid-connect/certs"
+          - name: "KEYCLOAKSEEDING__REALMS__0__IDENTITYPROVIDERS__0__CONFIG__AUTHORIZATIONURL"
+            value: "{{ .Values.realmSeeding.sharedidp }}/auth/realms/CX-Operator/protocol/openid-connect/auth"
+          - name: "KEYCLOAKSEEDING__REALMS__0__IDENTITYPROVIDERS__0__CONFIG__LOGOUTURL"
+            value: "{{ .Values.realmSeeding.sharedidp }}/auth/realms/CX-Operator/protocol/openid-connect/logout"
+
+          #############################
+          ## EXTRA SERVICE ACCOUNTS
+          #############################
+
+          {{- $index := 100 }}
+          {{- $secret := include "centralidp.secret.extraServiceAccounts" . -}}
+          {{- range .Values.realmSeeding.extraServiceAccounts.clientSecretsAndBpn }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__{{ $index }}__CLIENTID"
+            value: {{ .clientId | quote }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__CLIENTS__{{ $index }}__SECRET"
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret }}
+                key: {{ .clientId }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__{{ $index }}__USERNAME"
+            value: {{ printf "service-account-%s" .clientId | quote }}
+          - name: "KEYCLOAKSEEDING__REALMS__0__USERS__{{ $index }}__ATTRIBUTES__0__VALUES__0"
+            value: {{ .bpn | quote }}
+          {{- $index = add1 $index }}
+          {{- end }}
+
         ports:
         - name: http
-          containerPort: {{ .Values.seeding.portContainer }}
+          containerPort: {{ .Values.realmSeeding.portContainer }}
           protocol: TCP
         resources:
-          {{- toYaml .Values.seeding.resources | nindent 10 }}
+          {{- toYaml .Values.realmSeeding.resources | nindent 10 }}
         volumeMounts:
-        {{- toYaml .Values.seeding.extraVolumeMounts | nindent 10 }}
+          - name: realms
+            mountPath: "app/realms"
       initContainers:
-      {{- toYaml .Values.seeding.initContainers | nindent 8 }}
+        - name: init-cx-central
+          image: {{ .Values.realmSeeding.initContainer.image.name }}
+          imagePullPolicy: {{ .Values.realmSeeding.initContainer.image.pullPolicy }}
+          command:
+            - sh
+          args:
+            - -c
+            - |
+              echo "Copying CX Central realm..."
+              cp -R /import/catenax-central/realms/* /app/realms
+          volumeMounts:
+          - name: realms
+            mountPath: "app/realms"
+        - name: wait-for-keycloak
+          image: appropriate/curl:latest
+          {{ if .Values.realmSeeding.keycloakServiceTls -}}
+          command: [ 'sh', '-c', 'until curl -s https://{{ template "centralidp.fullname" . }}:{{ .Values.realmSeeding.keycloakServicePort }}/auth/realms/master/.well-known/openid-configuration; do echo waiting for other pod; sleep 2; done;' ]
+          {{- else -}}
+          command: [ 'sh', '-c', 'until curl -s http://{{ template "centralidp.fullname" . }}:{{ .Values.realmSeeding.keycloakServicePort }}/auth/realms/master/.well-known/openid-configuration; do echo waiting for other pod; sleep 2; done;' ]
+          {{- end }}
+          imagePullPolicy: IfNotPresent
       volumes:
-      {{- toYaml .Values.seeding.extraVolumes | nindent 8 }}
+        - name: realms
+          emptyDir: {}
+      automountServiceAccountToken: false
 {{- end -}}

charts/centralidp/templates/secret-base-service-accounts.yaml

@@ -0,0 +1,31 @@
+{{- /*
+* Copyright (c) 2024 Contributors to the Eclipse Foundation
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+*/}}
+
+{{- if and (.Values.realmSeeding.enabled) (not .Values.realmSeeding.serviceAccounts.existingSecret) -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "centralidp.secret.serviceAccounts" . }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+stringData:
+  {{- range .Values.realmSeeding.serviceAccounts.clientSecrets }}
+  {{ .clientId }}: {{ .clientSecret | default "changeme" | quote }}
+  {{- end }}
+{{- end -}}