diff --git a/docs/profiles.md b/docs/profiles.md index 1b719d72a..6d35577c5 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -148,3 +148,23 @@ that are potentially useful are: ```bash git config -f .gitmodules --unset submodule.xxx.branch ``` + +## Private repository profiles + +To use private repositories as submodules, the pipeline must have access to +the repository. This can be achieved by configuring the git credentials in the +vault path `/secret/git` under a key named `credentials`. These credentials are +usually in the form of a personal access token (PAT), which is put in the URL, +which points to the base host of the repository, as in this example: +`https://user:pat@github.com`. The URL is set as the value for the +`credentials` key: + +![vault git credentials](vault-git-credentials.png) + +If private repositories across multiple GIT servers are used, the credentials +URL for each one must be listed in the value, separated by space. + +The recommended settings for the PAT are: + +- For `GitHub`: use fine-grained token, providing access only to the profile +repositories with read permissions for: `Contents`, `Commit statuses`, and `Metadata`. diff --git a/docs/vault-git-credentials.png b/docs/vault-git-credentials.png new file mode 100644 index 000000000..b0e92f617 Binary files /dev/null and b/docs/vault-git-credentials.png differ diff --git a/terraform/gitlab/ci-templates/k8s-cluster/.gitlab-ci.yml b/terraform/gitlab/ci-templates/k8s-cluster/.gitlab-ci.yml index bc807505a..b4546930a 100644 --- a/terraform/gitlab/ci-templates/k8s-cluster/.gitlab-ci.yml +++ b/terraform/gitlab/ci-templates/k8s-cluster/.gitlab-ci.yml @@ -38,6 +38,7 @@ variables: K8S_TEMPLATE_PATH: terraform/k8s TMP_TEMPLATES_DIR: /tmp/iac-templates TMP_GIT_REPO: /tmp/git-iac-templates + GIT_SUBMODULE_STRATEGY: none IGNORE_TF_DEPRECATION_WARNING: true KUBE_IN_CLUSTER_CONFIG: true VAULT_ADDR: http://vault-active.vault.svc.cluster.local:8200 @@ -57,13 +58,18 @@ cache: .source: script: + - export VAULT_TOKEN="$(vault write -field=token auth/${VAULT_AUTH_PATH}/login role=$VAULT_AUTH_ROLE jwt=$VAULT_ID_TOKEN)" + - export GIT_CREDENTIALS="$(vault kv get -field=credentials ${KV_SECRET_PATH}/git)" + - if [ "$GIT_CREDENTIALS" != "" ]; then echo "$GIT_CREDENTIALS" | tr ' ' '\n' > ~/.gitcredentials.store; fi + - git config --global credential.helper 'store --file ~/.gitcredentials.store' + - git submodule sync --recursive + - git submodule update --init --recursive - source $(which gitlab-terraform) - .gitlab/scripts/config-merge.sh - yq eval '.' $CONFIG_PATH/cluster-config.yaml -o=json > cluster-config.json - yq eval '.' $CONFIG_PATH/addons-vars.yaml -o=json > addons-vars.yaml - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./cluster-config.json); do export $var; done - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./addons-vars.yaml); do export $var; done - - export VAULT_TOKEN="$(vault write -field=token auth/${VAULT_AUTH_PATH}/login role=$VAULT_AUTH_ROLE jwt=$VAULT_ID_TOKEN)" - export ENV_VAULT_TOKEN="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/env_token)" - export $cloud_platform_client_secret_name="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/cloud_platform_client_secret)" - export GITLAB_CI_PAT="$(vault kv get -field=value ${KV_SECRET_PATH}/gitlab/gitlab_ci_pat)" @@ -71,8 +77,7 @@ cache: - export NETBIRD_K8S_SETUP_KEY="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/netbird_k8s_setup_key)" - export KUBERNETES_OIDC_CLIENT_ID="$(vault kv get -field=value ${KV_SECRET_PATH}/${CI_PROJECT_NAME}/kubernetes_oidc_client_id)" - source setcivars.sh - - echo "https://${PRIVATE_REPO_USER}:${PRIVATE_REPO_TOKEN}@${PRIVATE_REPO}" > ~/.gitcredentials.store - - git config --global credential.helper 'store --file ~/.gitcredentials.store' + - echo "https://${PRIVATE_REPO_USER}:${PRIVATE_REPO_TOKEN}@${PRIVATE_REPO}" >> ~/.gitcredentials.store - git config --global advice.detachedHead false - source set-ansible-destroy-preq-vars.sh - export TERRAGRUNT_EXCLUDE_DIR="/dev/null" @@ -234,7 +239,7 @@ refresh-deploy-infra: script: - !reference [.source, script] - .gitlab/scripts/refresh-repo.sh $cluster_name $TMP_GIT_REPO $TMP_TEMPLATES_DIR $GITLAB_CI_PAT $iac_terraform_modules_tag - - git checkout "$CI_COMMIT_REF_NAME" && git pull --recurse-submodules + - git checkout "$CI_COMMIT_REF_NAME" && git pull - !reference [.source, script] - terragrunt run-all apply --terragrunt-non-interactive -input=false - .gitlab/scripts/pushtorepo.sh $CI_PROJECT_PATH $CI_SERVER_HOST $CI_COMMIT_REF_NAME $GITOPS_BUILD_OUTPUT_DIR $GITLAB_CI_PAT $ARGO_CD_ROOT_APP_PATH diff --git a/terraform/gitops/generate-files/templates/mcm/values-mcm.yaml.tpl b/terraform/gitops/generate-files/templates/mcm/values-mcm.yaml.tpl index 972f64b23..afba1da6f 100644 --- a/terraform/gitops/generate-files/templates/mcm/values-mcm.yaml.tpl +++ b/terraform/gitops/generate-files/templates/mcm/values-mcm.yaml.tpl @@ -10,7 +10,7 @@ db: api: image: name: ghcr.io/pm4ml/connection-manager-api - version: v2.1.0 + version: v2.4.0 url: https://${mcm_fqdn} extraTLS: rootCert: @@ -76,6 +76,8 @@ ui: clientId: ${oauth_key} clientSecretName: ${oauth_secret_secret} clientSecretKey: ${oauth_secret_secret_key} + image: + version: 1.8.4 ingress: %{ if istio_create_ingress_gateways ~} diff --git a/terraform/gitops/generate-files/templates/pm4ml/values-pm4ml.yaml.tpl b/terraform/gitops/generate-files/templates/pm4ml/values-pm4ml.yaml.tpl index d5f654134..c19fc9a31 100644 --- a/terraform/gitops/generate-files/templates/pm4ml/values-pm4ml.yaml.tpl +++ b/terraform/gitops/generate-files/templates/pm4ml/values-pm4ml.yaml.tpl @@ -129,7 +129,7 @@ scheme-adapter: DFSP_ID: *dfspId CACHE_URL: redis://${redis_host}:${redis_port} AUTO_ACCEPT_QUOTES: false - AUTO_ACCEPT_PARTY: false + AUTO_ACCEPT_PARTY: ${auto_accept_party} AUTO_ACCEPT_R2P_PARTY: false AUTO_ACCEPT_R2P_BUSINESS_QUOTES: false AUTO_ACCEPT_R2P_DEVICE_OTP: false diff --git a/terraform/gitops/pm4ml/pm4ml.tf b/terraform/gitops/pm4ml/pm4ml.tf index e06d894aa..b805ca38f 100644 --- a/terraform/gitops/pm4ml/pm4ml.tf +++ b/terraform/gitops/pm4ml/pm4ml.tf @@ -63,6 +63,7 @@ module "generate_pm4ml_files" { pm4ml_external_switch_client_secret_vault_value = "value" istio_external_gateway_name = var.istio_external_gateway_name cert_man_vault_cluster_issuer_name = var.cert_man_vault_cluster_issuer_name + auto_accept_party = each.value.auto_accept_party enable_sdk_bulk_transaction_support = each.value.enable_sdk_bulk_transaction_support kafka_host = "kafka" kafka_port = "9092" @@ -233,6 +234,11 @@ variable "pm4ml_external_switch_client_secret" { default = "pm4ml-external-switch-client-secret" } +variable "auto_accept_party" { + type = bool + description = "auto_accept_party" + default = false +} variable "enable_sdk_bulk_transaction_support" { type = bool description = "enable_sdk_bulk_transaction_support" diff --git a/terraform/k8s/default-config/cluster-config.yaml b/terraform/k8s/default-config/cluster-config.yaml index 67e0539ee..1606bbeca 100644 --- a/terraform/k8s/default-config/cluster-config.yaml +++ b/terraform/k8s/default-config/cluster-config.yaml @@ -8,8 +8,8 @@ cloud_platform_client_secret_name: AWS_SECRET_ACCESS_KEY k8s_cluster_module: base-k8s cloud_region: eu-west-1 k8s_cluster_type: microk8s -ansible_collection_tag: v5.3.9-rc01 -iac_terraform_modules_tag: v5.3.9-rc02 +ansible_collection_tag: v5.4.0-rc2 +iac_terraform_modules_tag: v5.4.0-rc2 letsencrypt_email: test@mojalabs.io dns_zone_force_destroy: true longhorn_backup_object_store_destroy: true diff --git a/terraform/k8s/default-config/mojaloop-vars.yaml b/terraform/k8s/default-config/mojaloop-vars.yaml index ba8599513..cbfa03602 100644 --- a/terraform/k8s/default-config/mojaloop-vars.yaml +++ b/terraform/k8s/default-config/mojaloop-vars.yaml @@ -1,6 +1,6 @@ bulk_enabled: false third_party_enabled: false -mcm_chart_version: 1.1.0 +mcm_chart_version: 1.2.4 mojaloop_chart_version: 16.0.0 ttk_testcases_tag: 16.1.0 currency: ${currency} diff --git a/terraform/k8s/default-config/pm4ml-vars.yaml b/terraform/k8s/default-config/pm4ml-vars.yaml index d736644a9..977d12353 100644 --- a/terraform/k8s/default-config/pm4ml-vars.yaml +++ b/terraform/k8s/default-config/pm4ml-vars.yaml @@ -6,6 +6,7 @@ pm4ml_external_switch_oidc_token_route: realms/dfsps/protocol/openid-connect/tok pm4ml_external_switch_client_secret_vault_path: "mcmdev_client_secret" pm4ml_ttk_enabled: true ttk_testcases_tag: v16.1.0 +auto_accept_party: false enable_sdk_bulk_transaction_support: false opentelemetry_enabled: false opentelemetry_namespace_filtering_enable: false