Merge branch 'master' into enableSwiftV2_SF_CNSRequestIPs

Signed-off-by: Kshitija Murudi <kmurudi@microsoft.com>
Azure · Feb 8, 2024 · 7a2b63d · 7a2b63d
2 parents ebf6640 + fea43a3
commit 7a2b63d
Show file tree

Hide file tree

Showing 93 changed files with 5,563 additions and 659 deletions.
diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml
@@ -38,22 +38,22 @@ jobs:
           echo ${{ secrets.AZURE_DEVOPS_EXT_PAT }} | az devops login --org ${{ secrets.AZURE_PIPELINE_ORG }}
 
           echo "Sanity check recently triggered Merge Queue Pipeline runs"
-          az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --reason individualCI --top 10 --output json | jq -r .[].sourceBranch
-          status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status`
+          az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --reason individualCI --top 10 --query-order QueueTimeDesc --output json | jq -r .[].sourceBranch
+          status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status`
           echo "Triggered CI Status - $status"
           echo "Branch Ref - $GITHUB_REF"
 
           echo "Checking for AZP triggered CI for 60s"
           end=$((SECONDS+60)) # Stop checking if not queued within a minute
           while [ $SECONDS -lt $end ]; do
+            echo "Waiting for 5 seconds for AZP to trigger run and show inProgress or notStarted"
+            sleep 5s
+            status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status`
+            echo "Current CI Status - $status"
             if [ $status = 'inProgress' ] || [ $status = 'notStarted' ]; then
               echo "AZP triggered pipeline queued successfully"
               exit 0
             fi
-            echo "Waiting for 15 seconds for AZP to trigger run and show inProgress or notStarted"
-            sleep 15s
-            status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status`
-            echo "Current CI Status - $status"
           done
           echo "AZP did not trigger CI"
 
@@ -66,7 +66,7 @@ jobs:
           while [ $SECONDS -lt $end ]; do
             echo "Waiting for 5 seconds for pipeline to show inProgress or notStarted on AZP"
             sleep 5s
-            status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status`
+            status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status`
             echo "Current CI Status - $status"
             if [ $status = 'inProgress' ] || [ $status = 'notStarted' ]; then
               echo "Manually triggered pipeline queued successfully"

diff --git a/.gitignore b/.gitignore
@@ -25,16 +25,13 @@ controller-gen
 build/tools/bin
 npm/debug/http
 
-# certificates
-*/**/certs/
-*.crt
-*.pem
-*.srl
-
 go.work*
 
 # scale-test
 test/scale/generated/*
 
 # test env file
 *.env
+
+# omit e2e bin
+test/e2e/bin/*
diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -8,6 +8,7 @@ parameters:
   os: "linux"
   arch: ""
   osSKU: Ubuntu
+  hubbleEnabled: false
 
 # Condition confirms that:
 # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success.
@@ -48,38 +49,80 @@ stages:
       - setup
     displayName: "Cilium Test - ${{ parameters.name }}"
     jobs:
-      - job: deploy_cilium_components
-        steps:
-          - task: AzureCLI@1
-            displayName: "Install Cilium, CNS, and ip-masq-agent"
-            inputs:
-              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -ex
-                az extension add --name aks-preview
-                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
-                ls -lah
-                pwd
-                kubectl cluster-info
-                kubectl get po -owide -A
+      - ${{if eq(parameters.hubbleEnabled, false)}}:
+        - job: deploy_cilium_components
+          displayName: Deploy Cilium
+          steps:
+            - task: AzureCLI@1
+              displayName: "Install Cilium, CNS, and ip-masq-agent"
+              inputs:
+                azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+                scriptLocation: "inlineScript"
+                scriptType: "bash"
+                addSpnToEnvironment: true
+                inlineScript: |
+                  set -ex
+                  az extension add --name aks-preview
+                  make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
+                  ls -lah
+                  pwd
+                  kubectl cluster-info
+                  kubectl get po -owide -A
+
+                  echo "deploy Cilium ConfigMap"
+                  kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
+                
+                  echo "install Cilium onto Overlay Cluster"
+                  kubectl apply -f test/integration/manifests/cilium/cilium-agent
+                  kubectl apply -f test/integration/manifests/cilium/cilium-operator
+                  
+                  
+                  echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster"
+                  # Passes Cilium image to daemonset and deployment
+                  envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
+                  envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
+                  kubectl get po -owide -A
+
+                  echo "Deploy Azure-CNS"
+                  sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                  kubectl get po -owide -A
+      - ${{if eq(parameters.hubbleEnabled, true)}}:
+        - job: deploy_cilium_components
+          displayName: Deploy Cilium with Hubble
+          steps:
+            - task: AzureCLI@1
+              displayName: "Install Cilium, CNS, and ip-masq-agent"
+              inputs:
+                azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+                scriptLocation: "inlineScript"
+                scriptType: "bash"
+                addSpnToEnvironment: true
+                inlineScript: |
+                  set -ex
+                  az extension add --name aks-preview
+                  make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
+                  ls -lah
+                  pwd
+                  kubectl cluster-info
+                  kubectl get po -owide -A
 
-                echo "deploy Cilium ConfigMap"
-                kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
-                echo "install Cilium onto Overlay Cluster"
-                kubectl apply -f test/integration/manifests/cilium/cilium-agent
-                kubectl apply -f test/integration/manifests/cilium/cilium-operator
-                echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster"
-                # Passes Cilium image to daemonset and deployment
-                envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
-                envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
-                kubectl get po -owide -A
+                  echo "deploy Cilium ConfigMap"
+                  kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-config/cilium-config-hubble.yaml
+                  
+                  echo "install Cilium onto Overlay Cluster with hubble enabled"
+                  kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-agent/files
+                  kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-operator/files
+                  
+                  echo "install Cilium v1.14.4 onto Overlay Cluster"
+                  # Passes Cilium image to daemonset and deployment
+                  envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl | kubectl apply -f -
+                  envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl | kubectl apply -f -
+                  kubectl get po -owide -A
 
-                echo "Deploy Azure-CNS"
-                sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
-                kubectl get po -owide -A
+                  echo "Deploy Azure-CNS"
+                  sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)
+                  kubectl get po -owide -A
+          
       - job: deploy_pods
         condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) )
         displayName: "Scale Test"

diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml
@@ -119,7 +119,7 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
-      - ${{ if eq(parameters.dualstack, true) }}:
+      - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
         - template: ../k8s-e2e/k8s-e2e-step-template.yaml
           parameters:
             testName: DualStack Test
@@ -130,3 +130,14 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
+      - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: DualStack Test|Cilium
+            name: DualStack
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[Feature:IPv6DualStack\]'
+            ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml
@@ -162,6 +162,15 @@ stages:
       nodeCount: ${NODE_COUNT_CILIUM}
       vmSize: ${VM_SIZE_CILIUM}
 
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay_hubble
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cil-over-hub"
+      hubbleEnabled: true
+      nodeCount: ${NODE_COUNT_CILIUM}
+      vmSize: ${VM_SIZE_CILIUM}
+
   - template: cilium/cilium-overlay-load-test-template.yaml
     parameters:
       name: cilium_overlay_mariner
@@ -269,6 +278,7 @@ stages:
     condition: always()
     dependsOn:
       - cilium_overlay
+      - cilium_overlay_hubble
       - cilium_overlay_mariner
       - cilium_overlay_arm
       - cilium_overlay_rdma
@@ -292,6 +302,9 @@ stages:
             cilium_overlay:
               name: cilium_overlay
               clusterName: "cilium-over"
+            cilium_overlay_hubble:
+               name: cilium_overlay_hubble
+               clusterName: "cil-over-hub"
             cilium_overlay_mariner:
               name: cilium_overlay_mariner
               clusterName: "cil-over-mar"

diff --git a/.pipelines/networkobservability/e2e.yaml b/.pipelines/networkobservability/e2e.yaml
@@ -0,0 +1,29 @@
+pr: none
+
+stages:
+  - stage: Run_E2E
+    displayName: Run E2E
+    jobs:
+      - job: Run_E2E
+        displayName: Run Network Observability E2E
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+
+        steps:
+          - task: AzureCLI@2
+            displayName: "Verify E2E Manifests"
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                go test -v ./test/e2e/manifests/*.go
+
+          - task: AzureCLI@2
+            displayName: "Run Network Observability E2E"
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                go test -v ./test/e2e/scenarios/hubble/*.go -timeout 30m
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
@@ -420,6 +420,30 @@ stages:
         k8sVersion: ""
         dependsOn: "test"
 
+    # Cilium Dualstack Overlay E2E tests
+    - template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
+      parameters:
+        name: "cilium_dualstackoverlay_e2e"
+        displayName: Cilium on AKS DualStack Overlay
+        os: linux
+        clusterType: dualstack-byocni-nokubeproxy-up
+        clusterName: "cildsovere2e"
+        vmSize: Standard_B2ms
+        k8sVersion: ""
+        dependsOn: "test"
+
+        # Cilium Overlay with hubble E2E tests
+    - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
+      parameters:
+        name: "cilium_h_overlay_e2e"
+        displayName: Cilium on AKS Overlay with Hubble
+        clusterType: overlay-byocni-nokubeproxy-up
+        clusterName: "cilwhleovere2e"
+        vmSize: Standard_B2ms
+        k8sVersion: ""
+        dependsOn: "test"
+        testHubble: true
+
     # Azure Overlay E2E tests
     - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
       parameters:
@@ -505,6 +529,7 @@ stages:
         - aks_swift_vnetscale_e2e
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
+        - cilium_dualstackoverlay_e2e
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
       jobs:
@@ -550,6 +575,9 @@ stages:
               dualstackoverlay_e2e:
                 name: dualstackoverlay_e2e
                 clusterName: "dsovere2e"
+              cilium_dualstackoverlay_e2e:
+                name: cilium_dualstackoverlay_e2e
+                clusterName: "cildsovere2e"
           steps:
             - template: templates/delete-cluster.yaml
               parameters:
@@ -584,9 +612,11 @@ stages:
         - aks_swift_e2e
         - cilium_e2e
         - cilium_overlay_e2e
+        - cilium_h_overlay_e2e
         - aks_ubuntu_22_linux_e2e
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
+        - cilium_dualstackoverlay_e2e
       jobs:
         - job: delete_remote_artifacts
           displayName: Delete remote artifacts

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -52,29 +52,13 @@ steps:
       pwd
       kubectl cluster-info
       kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true
+      sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
 
   - script: |
-      echo "Logs will be available as a build artifact"
-      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
-      echo $ARTIFACT_DIR
-      sudo rm -rf $ARTIFACT_DIR
-      sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration/logs/* $ARTIFACT_DIR
-    name: "GetLogs"
-    displayName: "Get logs"
-    condition: always()
-
-  - task: PublishBuildArtifacts@1
-    inputs:
-      artifactName: aks-swift-output
-      pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output"
-    condition: always()
-
-  - script: |
+      kubectl get po -owide -A
       echo "Run Service Conformance E2E"
       export PATH=${PATH}:/usr/local/bin/gsutil
       KUBECONFIG=~/.kube/config kubetest2 noop \
@@ -98,7 +82,7 @@ steps:
     retryCountOnTaskFailure: 3
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
-  
+
   - script: |
       cd hack/scripts
       chmod +x async-delete-test.sh
@@ -108,12 +92,3 @@ steps:
       fi
     name: "testAsyncDelete"
     displayName: "Verify Async Delete when CNS is down"
-
-  - script: |
-      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
-      echo $ARTIFACT_DIR
-      sudo rm -rf $ARTIFACT_DIR
-      sudo rm -rf test/integration/logs
-    name: "Cleanupartifactdir"
-    displayName: "Cleanup artifact dir"
-    condition: always()