Swiftv2 e2e tests (#2626)

* tmp commit to test whether the cluster can be set up.

* Resolved merge conflicts.
Update the route.

* Fix the route.

* Remove unnecessary dependency.

* Modify the cluster creation.

* Swift v2 e2e test structure.

* Add variables.

* Fix the format and make it a valid yaml.

* Add datapath tests for swiftv2.

* Link the tests with the pipeline steps.

* Update the testing file name.

* Remove unnecessary comment.

* Fix the pod name

* Sleep before checking the pod.

* Check the pni after baking time.

* Update the vnet name.

* Update the variables value.

* Update the sub tests use.

* Update the cluster name from mt prefix to mta

* Add the vm size.

* Update the token.

* Update the dummy cluster's vnet.

* Add a connection test.

* Update the test file name.

* Add one more pod with delegation network.

* Start 2 mtpod.

* Add the test for mtpod to mtpod connection.

* Include the commented out code for swiftv2 datapath test.

* Update the yaml file format.

* Update the command to run the swiftv2 tests.

* Move tests to the code file.

* Update the dummy cluster since the previous one has been auto deleted.

* Update the IP of the pod.

* Update the host IP in the test part too.

* Separate different test stages.

* Use mtpnc to grep IP.

* Update k to kubectl

* Add mtpnc detailed info.

* Remove the wrong comments.

* Use the script to run the test for now.

* Remove unnecessary variables.

* Update the cluster creation.

* Update parameters.

* Remove unnecessary parameters.

* Update the testing code part to match the script way.

* Remove unnecessary node part.

* Remove the tests from script part.

* Resolve the lint complaint.

* Resume the "Remove AKS Engine storage account usage"

.pipelines/multitenancy/swiftv2-e2e-job-template.yaml

@@ -0,0 +1,84 @@
+parameters:
+  name: ""
+  displayName: ""
+  clusterType: ""
+  clusterName: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
+  nodePoolName: ""
+
+stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../templates/create-cluster-swiftv2.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_SWIFTV2_CLUSTER_TEST) # Swiftv2 has a specific region requirements
+
+  - stage: ${{ parameters.name }}
+    # condition: and( succeeded(), not(eq(dependencies.dualstackoverlaye2e.result,'SucceededWithIssues')) ) # Cant use parameters in dependencies
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn:
+    - setup
+    - publish
+    - ${{ parameters.clusterName }}
+    variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Swiftv2 Multitenancy E2E Test Suite - (${{ parameters.name }})
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: swiftv2-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+
+      - template: ../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(ACN_TEST_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          dependsOn: ${{ parameters.name }}
+          datapath: true
+          dns: true
+          portforward: true
+          hostport: true
+          service: true
+
+      - job: failedE2ELogs
+        displayName: "Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+              cni: cniv2

.pipelines/multitenancy/swiftv2-e2e-step-template.yaml

@@ -0,0 +1,75 @@
+parameters:
+  name: ""
+  clusterName: ""
+  cni: cniv2
+  os: ""
+
+steps:
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - task: KubectlInstaller@0
+    inputs:
+      kubectlVersion: latest
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(ACN_TEST_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+        ls -lah
+        pwd
+        kubectl cluster-info
+        kubectl get po -owide -A
+        echo "Apply the pod network yaml to start the delegation"
+        less test/integration/manifests/swiftv2/podnetwork.yaml
+        envsubst '${SUBNET_TOKEN},${SUBNET_GUID},${SUBNET_RESOURCE_ID},${VNET_GUID}' < test/integration/manifests/swiftv2/podnetwork.yaml | kubectl apply -f -
+        echo "Check the podnetwork yaml file"
+        less test/integration/manifests/swiftv2/podnetwork.yaml
+        kubectl get pn
+        kubectl describe pn
+        echo "Apply the pod network instance yaml to reserve IP"
+        kubectl apply -f test/integration/manifests/swiftv2/pni.yaml
+        kubectl get pni
+        kubectl describe pni
+        export NODE_NAME_0="$(kubectl get nodes -o json | jq -r .items[0].metadata.name)"
+        echo $NODE_NAME_0
+        echo "Start the first pod using the reserved IP"
+        envsubst '$NODE_NAME_0' < test/integration/manifests/swiftv2/mtpod0.yaml | kubectl apply -f -
+        export NODE_NAME_1="$(kubectl get nodes -o json | jq -r .items[1].metadata.name)"
+        echo $NODE_NAME_1
+        echo "Start another pod using the reserved IP"
+        envsubst '$NODE_NAME_1' < test/integration/manifests/swiftv2/mtpod1.yaml | kubectl apply -f -
+        sleep 2m
+        kubectl get pod -o wide -A
+        sleep 2m
+        echo "Check pods after 4 minutes"
+        kubectl get po -owide -A
+        kubectl describe pni
+    name: "start_swiftv2_pods"
+    displayName: "Start Swiftv2 Pods"
+    env:
+      SUBNET_TOKEN: $(SUBNET_TOKEN)
+
+  - script: |
+      set -e
+      kubectl get po -owide -A
+      cd test/integration/swiftv2
+      echo "TestSwiftv2PodToPod and will run it after migration from scripts."
+      go test -count=1 swiftv2_test.go -timeout 3m -tags swiftv2 -run ^TestSwiftv2PodToPod$ -tags=swiftv2,integration -v
+    retryCountOnTaskFailure: 3
+    name: "Swiftv2_Tests_future_version"
+    displayName: "Swiftv2 Tests through code"
+

.pipelines/pipeline.yaml

@@ -504,6 +504,21 @@ stages:
         vmSize: Standard_B2ms
         dependsOn: "test"
 
+    # Swiftv2 E2E tests with multitenancy cluster start up
+    - template: multitenancy/swiftv2-e2e-job-template.yaml
+      parameters:
+        name: "swiftv2_e2e"
+        displayName: Swiftv2 Multitenancy
+        os: linux
+        clusterType: swiftv2-multitenancy-cluster-up
+        clusterName: "mtacluster"
+        nodePoolName: "mtapool"
+        vmSize: Standard_D4_v2
+        dependsOn: "test"
+        dummyClusterName: "swiftv2dummy"
+        dummyClusterType: "swiftv2-dummy-cluster-up"
+        dummyClusterDisplayName: Swiftv2 Multitenancy Dummy Cluster
+
     - stage: delete
       displayName: Delete Clusters
       condition: always()
@@ -518,6 +533,7 @@ stages:
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
         - cilium_dualstackoverlay_e2e
+        - swiftv2_e2e
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
       jobs:
@@ -566,6 +582,12 @@ stages:
               cilium_dualstackoverlay_e2e:
                 name: cilium_dualstackoverlay_e2e
                 clusterName: "cildsovere2e"
+              swiftv2_e2e:
+                name: swiftv2_e2e
+                clusterName: "mtcluster"
+              swiftv2_dummy_e2e:
+                name: swiftv2_dummy_e2e
+                clusterName: "swiftv2dummy"              
           steps:
             - template: templates/delete-cluster.yaml
               parameters:

.pipelines/templates/create-cluster-swiftv2.yaml

@@ -0,0 +1,34 @@
+parameters:
+  os: linux
+
+jobs:
+  - job: ${{ parameters.name }}
+    displayName: Cluster - ${{ parameters.name }}
+    steps:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(ACN_TEST_SERVICE_CONNECTION)
+          scriptLocation: "inlineScript"
+          scriptType: "bash"
+          addSpnToEnvironment: true
+          inlineScript: |
+            set -e
+            echo "Check az version"
+            az version
+            if ${{ lower(contains(parameters.clusterType, 'dualstack')) }}
+            then
+              echo "Install az cli extension preview"
+              az extension add --name aks-preview
+              az extension update --name aks-preview
+            fi
+            mkdir -p ~/.kube/
+            make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }}
+
+            make -C ./hack/aks ${{ parameters.clusterType }} \
+            AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) \
+            CLUSTER=${{ parameters.clusterName }} \
+            VM_SIZE=${{ parameters.vmSize }} OS=${{parameters.os}} \
+
+            echo "Cluster successfully created"
+        displayName: Cluster - ${{ parameters.clusterType }}
+        continueOnError: ${{ contains(parameters.clusterType, 'dualstack') }}

hack/aks/Makefile

@@ -233,6 +233,28 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster
 		--yes
 	@$(MAKE) set-kubeconf
 
+swiftv2-multitenancy-cluster-up: rg-up
+	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
+		--network-plugin azure \
+		--network-plugin-mode overlay \
+		--kubernetes-version 1.28 \
+		--nodepool-name "mtapool" \
+		--node-vm-size Standard_D4_v2 \
+		--node-count 2 \
+		--nodepool-tags fastpathenabled=true \
+		--no-ssh-key \
+		--yes
+	@$(MAKE) set-kubeconf
+
+swiftv2-dummy-cluster-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster
+	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
+		--network-plugin azure \
+		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
+		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
+		--no-ssh-key \
+		--yes
+	@$(MAKE) set-kubeconf
+
 # The below Vnet Scale clusters are currently only in private preview and available with Kubernetes 1.28
 # These AKS clusters can only be created in a limited subscription listed here:
 # https://dev.azure.com/msazure/CloudNativeCompute/_git/aks-rp?path=/resourceprovider/server/microsoft.com/containerservice/flags/network_flags.go&version=GBmaster&line=134&lineEnd=135&lineStartColumn=1&lineEndColumn=1&lineStyle=plain&_a=contents

test/integration/manifests/swiftv2/mtpod0.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    kubernetes.azure.com/pod-network: aksswiftvnetv20425
+    kubernetes.azure.com/pod-network-instance: pni1
+  name: mtpod0
+  namespace: default
+spec:
+  containers:
+  - image: nicolaka/netshoot:latest
+    imagePullPolicy: Always
+    name: mtpod0
+    command: ["/bin/bash"]
+    args: ["-c", "while true; do ping localhost; sleep 60;done"]
+    securityContext:
+       privileged: true
+    ports:
+    - containerPort: 80
+      protocol: TCP
+  nodeSelector:
+    kubernetes.io/hostname: $NODE_NAME_0

test/integration/manifests/swiftv2/mtpod1.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    kubernetes.azure.com/pod-network: aksswiftvnetv20425
+    kubernetes.azure.com/pod-network-instance: pni1
+  name: mtpod1
+  namespace: default
+spec:
+  containers:
+  - image: nicolaka/netshoot:latest
+    imagePullPolicy: Always
+    name: mtpod1
+    command: ["/bin/bash"]
+    args: ["-c", "while true; do ping localhost; sleep 60;done"]
+    securityContext:
+       privileged: true
+    ports:
+    - containerPort: 80
+      protocol: TCP
+  nodeSelector:
+    kubernetes.io/hostname: $NODE_NAME_1

test/integration/manifests/swiftv2/pni.yaml

@@ -0,0 +1,7 @@
+apiVersion: multitenancy.acn.azure.com/v1alpha1
+kind: PodNetworkInstance
+metadata:
+   name: pni1
+spec:
+   podnetwork: aksswiftvnetv20425
+   podIPReservationSize: 2

test/integration/manifests/swiftv2/podnetwork.yaml

@@ -0,0 +1,10 @@
+apiVersion: multitenancy.acn.azure.com/v1alpha1
+kind: PodNetwork
+metadata:
+  labels:
+    kubernetes.azure.com/override-subnet-token: $SUBNET_TOKEN
+  name: aksswiftvnetv20425
+spec:
+  subnetGUID: $SUBNET_GUID
+  subnetResourceID: $SUBNET_RESOURCE_ID
+  vnetGUID: $VNET_GUID