ci: migrate windows azure job to eks prow cluster

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
kubernetes-sigs · Jul 23, 2024 · 2051fc4 · 2051fc4
1 parent 555135f
commit 2051fc4
Show file tree

Hide file tree

Showing 11 changed files with 130 additions and 158 deletions.
diff --git a/Makefile b/Makefile
@@ -438,7 +438,8 @@ e2e-helm-deploy:
 		--set rotationPollInterval=30s \
 		--set tokenRequests[0].audience="aud1" \
 		--set tokenRequests[1].audience="aud2" \
-		--set tokenRequests[2].audience="conjur"
+		--set tokenRequests[2].audience="conjur" \
+		--set tokenRequests[3].audience="api://AzureADTokenExchange"
 
 .PHONY: e2e-helm-upgrade
 e2e-helm-upgrade:

diff --git a/test/bats/azure.bats b/test/bats/azure.bats
@@ -16,17 +16,17 @@ if [ $TEST_WINDOWS ]; then
   NODE_SELECTOR_OS=windows
 fi
 
-if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then
-    export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6)
-fi
+# if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then
+#     export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6)
+# fi
 
-export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e}
+export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e}
 export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1}
 export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""}
 export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"}
-export KEY_NAME=${KEYVAULT_KEY_NAME:-key1}
-export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a}
-export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"}
+# export KEY_NAME=${KEYVAULT_KEY_NAME:-key1}
+# export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a}
+# export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"}
 export LABEL_VALUE=${LABEL_VALUE:-"test"}
 export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
 
@@ -35,8 +35,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
 export API_VERSION=$(get_secrets_store_api_version)
 
 setup() {
-  if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then
-    echo "Error: Azure service principal is not provided" >&2
+  if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then
+    echo "Error: Azure managed identity id is not provided" >&2
     return 1
   fi
 }
@@ -55,15 +55,6 @@ setup() {
   kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE
 }
 
-@test "create azure k8s secret" {
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET}
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
-  assert_success
-}
-
 @test "deploy azure secretproviderclass crd" {
   envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f -
 
@@ -92,11 +83,11 @@ setup() {
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 }
 
-@test "CSI inline volume test with pod portability - read azure kv key from pod" {
-  result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-}
+# @test "CSI inline volume test with pod portability - read azure kv key from pod" {
+#   result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME)
+#   result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
+#   [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
+# }
 
 @test "CSI inline volume test with pod portability - unmount succeeds" {
   # On Linux a failure to unmount the tmpfs will block the pod from being
@@ -140,9 +131,9 @@ setup() {
   result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
+  # result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME)
+  # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
+  # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
 
   result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
@@ -180,12 +171,12 @@ setup() {
   run kubectl create ns test-ns
   assert_success
 
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns
-  assert_success
+  # run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns
+  # assert_success
 
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns
-  assert_success
+  # # label the node publish secret ref secret
+  # run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns
+  # assert_success
 
   envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f -
 
@@ -208,9 +199,9 @@ setup() {
   result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
+  # result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME)
+  # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
+  # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
 
   result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
@@ -234,12 +225,12 @@ setup() {
   run kubectl create ns negative-test-ns
   assert_success
 
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns
-  assert_success
+  # run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns
+  # assert_success
 
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns
-  assert_success
+  # # label the node publish secret ref secret
+  # run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns
+  # assert_success
 
   envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f -
   sleep 5
@@ -280,16 +271,16 @@ setup() {
   result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
+  # result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME)
+  # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
+  # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
 
   result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
+  # result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME)
+  # result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
+  # [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
 
   result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
@@ -310,57 +301,6 @@ setup() {
   assert_success
 }
 
-@test "Test auto rotation of mount contents and K8s secrets - Create deployment" {
-  run kubectl create ns rotation
-  assert_success
-
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation
-  assert_success
-
-  run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal
-  assert_success
-
-  run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret
-  assert_success
-
-  envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f -
-  envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f -
-
-  kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation
-
-  run kubectl get pod/secrets-store-inline-rotation -n rotation
-  assert_success
-}
-
-@test "Test auto rotation of mount contents and K8s secrets" {
-  result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
-  [[ "${result//$'\r'}" == "secret" ]]
-
-  result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
-  [[ "${result//$'\r'}" == "secret" ]]
-
-  run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated
-  assert_success
-
-  sleep 60
-
-  result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
-  [[ "${result//$'\r'}" == "rotated" ]]
-
-  result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
-  [[ "${result//$'\r'}" == "rotated" ]]
-
-  run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME}
-  assert_success
-
-  run az logout
-  assert_success
-}
-
 teardown_file() {
   archive_provider "app=csi-secrets-store-provider-azure" || true
   archive_info || true
@@ -369,7 +309,7 @@ teardown_file() {
   run kubectl delete namespace rotation
   run kubectl delete namespace test-ns
 
-  run kubectl delete secret secrets-store-creds
+  # run kubectl delete secret secrets-store-creds
 
   run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0
 }
diff --git a/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml
@@ -13,7 +13,7 @@ spec:
     - objectName: secretalias                    # name of the mounted content to sync. this could be the object name or object alias 
       key: username
   parameters:
-    usePodIdentity: "false"                      # [OPTIONAL] if not provided, will default to "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"               # the name of the KeyVault
     objects: |
       array:
@@ -22,8 +22,8 @@ spec:
           objectType: secret                     # object types: secret, key or cert
           objectAlias: secretalias
           objectVersion: $SECRET_VERSION         # [OPTIONAL] object versions, default to latest if empty
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault
diff --git a/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml
@@ -11,7 +11,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -20,11 +20,11 @@ spec:
           objectType: secret
           objectVersion: $SECRET_VERSION
           objectAlias: secretalias
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID"
 ---
 apiVersion: $API_VERSION
 kind: SecretProviderClass
@@ -39,7 +39,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -48,8 +48,8 @@ spec:
           objectType: secret
           objectVersion: $SECRET_VERSION
           objectAlias: secretalias
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID"
diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass.yaml
@@ -5,16 +5,16 @@ metadata:
 spec:
   provider: azure
   parameters:
-    usePodIdentity: "false"         # [OPTIONAL] if not provided, will default to "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
     objects: |
       array:
         - |
           objectName: $SECRET_NAME
           objectType: secret        # object types: secret, key or cert
           objectVersion: $SECRET_VERSION         # [OPTIONAL] object versions, default to latest if empty
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault
diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml
@@ -12,7 +12,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -21,11 +21,11 @@ spec:
           objectType: secret
           objectAlias: secretalias
           objectVersion: $SECRET_VERSION
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID"
 ---
 apiVersion: $API_VERSION
 kind: SecretProviderClass
@@ -41,7 +41,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -50,8 +50,8 @@ spec:
           objectType: secret
           objectAlias: secretalias
           objectVersion: $SECRET_VERSION
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+        # - |
+        #   objectName: $KEY_NAME
+        #   objectType: key
+        #   objectVersion: $KEY_VERSION
+    tenantId: "$AZURE_TENANT_ID"
diff --git a/test/bats/tests/azure/deployment-synck8s-azure.yaml b/test/bats/tests/azure/deployment-synck8s-azure.yaml
@@ -39,7 +39,7 @@ spec:
             readOnly: true
             volumeAttributes:
               secretProviderClass: "azure-sync"
-            nodePublishSecretRef:
-              name: secrets-store-creds
+            # nodePublishSecretRef:
+            #   name: secrets-store-creds
       nodeSelector:
         kubernetes.io/os: $NODE_SELECTOR_OS
diff --git a/test/bats/tests/azure/deployment-two-synck8s-azure.yaml b/test/bats/tests/azure/deployment-two-synck8s-azure.yaml
@@ -39,7 +39,7 @@ spec:
             readOnly: true
             volumeAttributes:
               secretProviderClass: "azure-sync"
-            nodePublishSecretRef:
-              name: secrets-store-creds
+            # nodePublishSecretRef:
+            #   name: secrets-store-creds
       nodeSelector:
         kubernetes.io/os: $NODE_SELECTOR_OS