From 2051fc499c0636c4e58be5b6932a33509ffe2dc4 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Mon, 22 Jul 2024 14:12:54 -0700 Subject: [PATCH] ci: migrate windows azure job to eks prow cluster Signed-off-by: Anish Ramasekar --- Makefile | 3 +- test/bats/azure.bats | 134 +++++------------- .../azure_synck8s_v1_secretproviderclass.yaml | 12 +- ...azure_v1_multiple_secretproviderclass.yaml | 24 ++-- .../azure/azure_v1_secretproviderclass.yaml | 12 +- .../azure_v1_secretproviderclass_ns.yaml | 24 ++-- .../tests/azure/deployment-synck8s-azure.yaml | 4 +- .../azure/deployment-two-synck8s-azure.yaml | 4 +- .../pod-azure-inline-volume-multiple-spc.yaml | 8 +- .../pod-secrets-store-inline-volume-crd.yaml | 4 +- test/scripts/run-e2e-windows.sh | 59 ++++++-- 11 files changed, 130 insertions(+), 158 deletions(-) diff --git a/Makefile b/Makefile index f3332f1bf..4ef518722 100644 --- a/Makefile +++ b/Makefile @@ -438,7 +438,8 @@ e2e-helm-deploy: --set rotationPollInterval=30s \ --set tokenRequests[0].audience="aud1" \ --set tokenRequests[1].audience="aud2" \ - --set tokenRequests[2].audience="conjur" + --set tokenRequests[2].audience="conjur" \ + --set tokenRequests[3].audience="api://AzureADTokenExchange" .PHONY: e2e-helm-upgrade e2e-helm-upgrade: diff --git a/test/bats/azure.bats b/test/bats/azure.bats index 7e585847b..ac689074a 100644 --- a/test/bats/azure.bats +++ b/test/bats/azure.bats @@ -16,17 +16,17 @@ if [ $TEST_WINDOWS ]; then NODE_SELECTOR_OS=windows fi -if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then - export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6) -fi +# if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then +# export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6) +# fi -export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e} +export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e} export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1} export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""} export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"} -export KEY_NAME=${KEYVAULT_KEY_NAME:-key1} -export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a} -export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"} +# export KEY_NAME=${KEYVAULT_KEY_NAME:-key1} +# export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a} +# export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"} export LABEL_VALUE=${LABEL_VALUE:-"test"} export NODE_SELECTOR_OS=$NODE_SELECTOR_OS @@ -35,8 +35,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS export API_VERSION=$(get_secrets_store_api_version) setup() { - if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then - echo "Error: Azure service principal is not provided" >&2 + if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then + echo "Error: Azure managed identity id is not provided" >&2 return 1 fi } @@ -55,15 +55,6 @@ setup() { kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE } -@test "create azure k8s secret" { - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - assert_success -} - @test "deploy azure secretproviderclass crd" { envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f - @@ -92,11 +83,11 @@ setup() { [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] } -@test "CSI inline volume test with pod portability - read azure kv key from pod" { - result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] -} +# @test "CSI inline volume test with pod portability - read azure kv key from pod" { +# result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME) +# result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) +# [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] +# } @test "CSI inline volume test with pod portability - unmount succeeds" { # On Linux a failure to unmount the tmpfs will block the pod from being @@ -140,9 +131,9 @@ setup() { result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + # result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME) + # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) + # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -180,12 +171,12 @@ setup() { run kubectl create ns test-ns assert_success - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns - assert_success + # run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns + # assert_success - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns - assert_success + # # label the node publish secret ref secret + # run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns + # assert_success envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f - @@ -208,9 +199,9 @@ setup() { result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + # result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME) + # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) + # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -234,12 +225,12 @@ setup() { run kubectl create ns negative-test-ns assert_success - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns - assert_success + # run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns + # assert_success - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns - assert_success + # # label the node publish secret ref secret + # run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns + # assert_success envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f - sleep 5 @@ -280,16 +271,16 @@ setup() { result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + # result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME) + # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) + # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + # result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME) + # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) + # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -310,57 +301,6 @@ setup() { assert_success } -@test "Test auto rotation of mount contents and K8s secrets - Create deployment" { - run kubectl create ns rotation - assert_success - - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation - assert_success - - run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal - assert_success - - run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret - assert_success - - envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f - - envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f - - - kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation - - run kubectl get pod/secrets-store-inline-rotation -n rotation - assert_success -} - -@test "Test auto rotation of mount contents and K8s secrets" { - result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias) - [[ "${result//$'\r'}" == "secret" ]] - - result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d) - [[ "${result//$'\r'}" == "secret" ]] - - run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated - assert_success - - sleep 60 - - result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias) - [[ "${result//$'\r'}" == "rotated" ]] - - result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d) - [[ "${result//$'\r'}" == "rotated" ]] - - run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} - assert_success - - run az logout - assert_success -} - teardown_file() { archive_provider "app=csi-secrets-store-provider-azure" || true archive_info || true @@ -369,7 +309,7 @@ teardown_file() { run kubectl delete namespace rotation run kubectl delete namespace test-ns - run kubectl delete secret secrets-store-creds + # run kubectl delete secret secrets-store-creds run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0 } diff --git a/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml index 55e2979d5..16e70c801 100644 --- a/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml @@ -13,7 +13,7 @@ spec: - objectName: secretalias # name of the mounted content to sync. this could be the object name or object alias key: username parameters: - usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault objects: | array: @@ -22,8 +22,8 @@ spec: objectType: secret # object types: secret, key or cert objectAlias: secretalias objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" # the tenant ID of the KeyVault + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault diff --git a/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml index 5655b1eeb..37b923ff9 100644 --- a/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml @@ -11,7 +11,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -20,11 +20,11 @@ spec: objectType: secret objectVersion: $SECRET_VERSION objectAlias: secretalias - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" --- apiVersion: $API_VERSION kind: SecretProviderClass @@ -39,7 +39,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -48,8 +48,8 @@ spec: objectType: secret objectVersion: $SECRET_VERSION objectAlias: secretalias - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass.yaml index 20ac1507c..cc53265dd 100644 --- a/test/bats/tests/azure/azure_v1_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_v1_secretproviderclass.yaml @@ -5,7 +5,7 @@ metadata: spec: provider: azure parameters: - usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault objects: | array: @@ -13,8 +13,8 @@ spec: objectName: $SECRET_NAME objectType: secret # object types: secret, key or cert objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" # the tenant ID of the KeyVault + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml index d48d5e588..437ba095a 100644 --- a/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml +++ b/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml @@ -12,7 +12,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -21,11 +21,11 @@ spec: objectType: secret objectAlias: secretalias objectVersion: $SECRET_VERSION - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" --- apiVersion: $API_VERSION kind: SecretProviderClass @@ -41,7 +41,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -50,8 +50,8 @@ spec: objectType: secret objectAlias: secretalias objectVersion: $SECRET_VERSION - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + # - | + # objectName: $KEY_NAME + # objectType: key + # objectVersion: $KEY_VERSION + tenantId: "$AZURE_TENANT_ID" diff --git a/test/bats/tests/azure/deployment-synck8s-azure.yaml b/test/bats/tests/azure/deployment-synck8s-azure.yaml index 880931bb1..17014c101 100644 --- a/test/bats/tests/azure/deployment-synck8s-azure.yaml +++ b/test/bats/tests/azure/deployment-synck8s-azure.yaml @@ -39,7 +39,7 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-sync" - nodePublishSecretRef: - name: secrets-store-creds + # nodePublishSecretRef: + # name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/deployment-two-synck8s-azure.yaml b/test/bats/tests/azure/deployment-two-synck8s-azure.yaml index 8c2d05e81..076fb6948 100644 --- a/test/bats/tests/azure/deployment-two-synck8s-azure.yaml +++ b/test/bats/tests/azure/deployment-two-synck8s-azure.yaml @@ -39,7 +39,7 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-sync" - nodePublishSecretRef: - name: secrets-store-creds + # nodePublishSecretRef: + # name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml b/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml index 53274cc4d..4cb2798ae 100644 --- a/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml +++ b/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml @@ -36,15 +36,15 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-spc-0" - nodePublishSecretRef: - name: secrets-store-creds + # nodePublishSecretRef: + # name: secrets-store-creds - name: secrets-store-inline-1 csi: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: secretProviderClass: "azure-spc-1" - nodePublishSecretRef: - name: secrets-store-creds + # nodePublishSecretRef: + # name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml b/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml index 8223e8b74..a949357ce 100644 --- a/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml +++ b/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml @@ -22,7 +22,7 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure" - nodePublishSecretRef: - name: secrets-store-creds + # nodePublishSecretRef: + # name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/scripts/run-e2e-windows.sh b/test/scripts/run-e2e-windows.sh index a214d1b99..871d799db 100755 --- a/test/scripts/run-e2e-windows.sh +++ b/test/scripts/run-e2e-windows.sh @@ -21,21 +21,22 @@ set -o pipefail : "${REGISTRY:?Environment variable empty or not defined.}" readonly CLUSTER_NAME="${CLUSTER_NAME:-sscsi-e2e-$(openssl rand -hex 2)}" +readonly KEYVAULT_NAME="secrets-store-csi-e2e" parse_cred() { grep -E -o "\b$1[[:blank:]]*=[[:blank:]]*\"[^[:space:]\"]+\"" | cut -d '"' -f 2 } get_random_region() { - local REGIONS=("eastus" "eastus2" "southcentralus" "westeurope" "uksouth") + local REGIONS=("eastus" "eastus2" "uksouth") echo "${REGIONS[${RANDOM} % ${#REGIONS[@]}]}" } cleanup() { echo "Deleting the AKS cluster ${CLUSTER_NAME}" # login again because the bats tests might have logged out after rotating the secret for testing - az login --service-principal -u "${client_id}" -p "${client_secret}" --tenant "${tenant_id}" > /dev/null - az account set --subscription "${subscription_id}" > /dev/null + az login --service-principal -u "${AZURE_CLIENT_ID}" --t "${AZURE_TENANT_ID}" --federated-token "$(cat "${AZURE_FEDERATED_TOKEN_FILE}")" > /dev/null + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" > /dev/null az group delete --name "${CLUSTER_NAME}" --yes --no-wait || true } trap cleanup EXIT @@ -45,8 +46,8 @@ main() { curl -sL https://aka.ms/InstallAzureCLIDeb | bash > /dev/null echo "Logging into Azure" - az login --service-principal -u "${client_id}" -p "${client_secret}" --tenant "${tenant_id}" > /dev/null - az account set --subscription "${subscription_id}" > /dev/null + az login --service-principal -u "${AZURE_CLIENT_ID}" --t "${AZURE_TENANT_ID}" --federated-token "$(cat "${AZURE_FEDERATED_TOKEN_FILE}")" > /dev/null + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" > /dev/null LOCATION=$(get_random_region) echo "Creating AKS cluster ${CLUSTER_NAME} in ${LOCATION}" @@ -58,6 +59,7 @@ main() { --node-vm-size Standard_DS3_v2 \ --enable-managed-identity \ --network-plugin azure \ + --enable-oidc-issuer \ --generate-ssh-keys > /dev/null echo "Adding windows nodepool" @@ -71,21 +73,50 @@ main() { az aks get-credentials --resource-group "${CLUSTER_NAME}" --name "${CLUSTER_NAME}" --overwrite-existing + # confirm the cluster is up and running + kubectl get nodes -o wide + kubectl get pods -A + if [[ "${REGISTRY}" =~ \.azurecr\.io ]]; then az acr login --name "${REGISTRY}" fi + AKS_CLUSTER_OIDC_ISSUER_URL=$(az aks show -g "${CLUSTER_NAME}" -n "${CLUSTER_NAME}" --query "oidcIssuerProfile.issuerUrl" -otsv) + # Create managed identity that'll be used by the provider to access keyvault + echo "Creating managed identity" + user_assigned_identity_name="sscsi-e2e-$(openssl rand -hex 2)" + az identity create --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" > /dev/null + export IDENTITY_CLIENT_ID=$(az identity show --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" --query 'clientId' -otsv) + IDENTITY_OBJECT_ID=$(az identity show --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" --query 'principalId' -otsv) + + # Create the federated identity credential (FIC) for the managed identity + echo "Creating federated identity credential for default:default" + az identity federated-credential create --name "kubernetes-federated-credential-default" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:default:default" > /dev/null + + echo "Creating federated identity credential for test-ns:default" + az identity federated-credential create --name "kubernetes-federated-credential-test-ns" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:test-ns:default" > /dev/null + + echo "Creating federated identity credential for negative-test-ns:default" + az identity federated-credential create --name "kubernetes-federated-credential-negative-test-ns" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:negative-test-ns:default" > /dev/null + + # Assigning the managed identity the necessary permissions to access the keyvault + echo "Assigning managed identity permissions to get secrets from keyvault" + az keyvault set-policy --name "${KEYVAULT_NAME}" --secret-permissions get --object-id "${IDENTITY_OBJECT_ID}" > /dev/null + # build the driver image and run e2e tests ALL_OS_ARCH=amd64 make e2e-test } -# for Prow we use the provided AZURE_CREDENTIALS file. -# the file is expected to be in toml format. -if [[ -n "${AZURE_CREDENTIALS:-}" ]]; then - subscription_id="$(parse_cred SubscriptionID < "${AZURE_CREDENTIALS}")" - tenant_id="$(parse_cred TenantID < "${AZURE_CREDENTIALS}")" - client_id="$(parse_cred ClientID < "${AZURE_CREDENTIALS}")" - client_secret="$(parse_cred ClientSecret < "${AZURE_CREDENTIALS}")" -fi - main