ci:Adding cilium dualstack overlay pipeline steps (#2418)

* adding cilium dualstack overlay pipeline steps * changing step order * remove dropgz step * add CNS instead of azurevnet * add v6 CNS state file test * remove nightly pipeline * removing nightly and adding v6 to cilium linux validate * fix func name * removing unneeded steps * addressing comments * remove 120 timeout * cluster type to dualstack-byocni-nokubeproxy-up * setting to INSTALL_OVERLAY * using make to get azure-ipam nad cns * fixing yaml in cil dual steps * update step * skipping bad step for cilium dualstack * update variable name * CNI type * remove file path * fixing CNI versions * removing log outputs * change step * adding cd in directory before running validate * hostport to false * further update * add cilium to dualstack test name
Azure · Feb 5, 2024 · 0ac2938 · 0ac2938
1 parent e735d6e
commit 0ac2938
Show file tree

Hide file tree

Showing 6 changed files with 372 additions and 6 deletions.
diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml
@@ -119,7 +119,7 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
-      - ${{ if eq(parameters.dualstack, true) }}:
+      - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
         - template: ../k8s-e2e/k8s-e2e-step-template.yaml
           parameters:
             testName: DualStack Test
@@ -130,3 +130,14 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
+      - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: DualStack Test|Cilium
+            name: DualStack
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[Feature:IPv6DualStack\]'
+            ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
@@ -420,6 +420,18 @@ stages:
         k8sVersion: ""
         dependsOn: "test"
 
+    # Cilium Dualstack Overlay E2E tests
+    - template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
+      parameters:
+        name: "cilium_dualstackoverlay_e2e"
+        displayName: Cilium on AKS DualStack Overlay
+        os: linux
+        clusterType: dualstack-byocni-nokubeproxy-up
+        clusterName: "cildsovere2e"
+        vmSize: Standard_B2ms
+        k8sVersion: ""
+        dependsOn: "test"
+
         # Cilium Overlay with hubble E2E tests
     - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
       parameters:
@@ -517,6 +529,7 @@ stages:
         - aks_swift_vnetscale_e2e
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
+        - cilium_dualstackoverlay_e2e
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
       jobs:
@@ -562,6 +575,9 @@ stages:
               dualstackoverlay_e2e:
                 name: dualstackoverlay_e2e
                 clusterName: "dsovere2e"
+              cilium_dualstackoverlay_e2e:
+                name: cilium_dualstackoverlay_e2e
+                clusterName: "cildsovere2e"
           steps:
             - template: templates/delete-cluster.yaml
               parameters:
@@ -600,6 +616,7 @@ stages:
         - aks_ubuntu_22_linux_e2e
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
+        - cilium_dualstackoverlay_e2e
       jobs:
         - job: delete_remote_artifacts
           displayName: Delete remote artifacts

diff --git a/...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml b/...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
@@ -0,0 +1,82 @@
+parameters:
+  name: ""
+  displayName: ""
+  clusterType: ""
+  clusterName: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
+
+stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../../templates/create-cluster.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement
+
+  - stage: ${{ parameters.name }}
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn:
+    - setup
+    - publish
+    - ${{ parameters.clusterName }}
+    variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Cilium Dualstack Overlay Test Suite - (${{ parameters.name }})
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: cilium-dualstackoverlay-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          cni: cilium
+          dependsOn: ${{ parameters.name }}
+          dualstack: true
+          dns: true
+          portforward: true
+          service: true
+
+      - job: failedE2ELogs
+        displayName: "Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}
+          - cni_${{ parameters.os }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cilium
diff --git a/...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml
@@ -0,0 +1,124 @@
+parameters:
+  name: ""
+  clusterName: ""
+
+steps:
+
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - task: KubectlInstaller@0
+    inputs:
+      kubectlVersion: latest
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+        ls -lah
+        pwd
+        kubectl cluster-info
+        kubectl get po -owide -A
+        echo "deploy Cilium ConfigMap"
+        kubectl apply -f test/integration/manifests/cilium/cilium-config-dualstack.yaml
+        echo "install Cilium ${CILIUM_VERSION_TAG}"
+        # Passes Cilium image to daemonset and deployment
+        envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
+        envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
+        # Use different file directories for nightly and current cilium version
+        kubectl apply -f test/integration/manifests/cilium/cilium-agent
+        kubectl apply -f test/integration/manifests/cilium/cilium-operator
+        kubectl get po -owide -A
+    name: "installCilium"
+    displayName: "Install Cilium on AKS Dualstack Overlay"
+
+  - script: |
+      echo "install cilium CLI"
+      if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then
+        echo "Cilium Agent Version ${BASH_REMATCH[0]}"
+        CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt)
+      else
+        echo "Cilium Agent Version ${CILIUM_VERSION_TAG}"
+        CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
+      fi
+      CLI_ARCH=amd64
+      if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
+      curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
+      sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
+      sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
+      rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
+      cilium status
+      cilium version
+    name: "installCiliumCLI"
+    displayName: "Install Cilium CLI"
+
+  - script: |
+      echo "Start Azilium E2E Tests on Overlay Cluster"
+      sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true
+    retryCountOnTaskFailure: 3
+    name: "aziliumTest"
+    displayName: "Run Azilium E2E on AKS Overlay"
+
+  - script: |
+      kubectl get pods -A
+      echo "Waiting < 2 minutes for cilium to be ready"
+      # Ensure Cilium is ready Xm\Xs
+      cilium status --wait --wait-duration 2m
+    retryCountOnTaskFailure: 3
+    name: "CiliumStatus"
+    displayName: "Cilium Status"
+
+  - script: |
+      echo "Run Cilium Connectivity Tests"
+      cilium status
+      cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption'
+    retryCountOnTaskFailure: 3
+    name: "ciliumConnectivityTests"
+    displayName: "Run Cilium Connectivity Tests"
+
+  - script: | 
+      set -e 
+      kubectl get po -owide -A 
+      cd test/integration/datapath 
+      echo "Dualstack Overlay Linux datapath IPv6 test" 
+      go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true 
+      echo "Dualstack Overlay Linux datapath IPv4 test" 
+      go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration 
+    retryCountOnTaskFailure: 3 
+    name: "DualStack_Overlay_Linux_Tests" 
+    displayName: "DualStack Overlay Linux Tests" 
+
+  - script: |
+      echo "validate pod IP assignment and check systemd-networkd restart"
+      kubectl get pod -owide -A
+      # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change.
+      # Saves 17 minutes
+      kubectl delete deploy -n cilium-test echo-external-node
+      cd test/integration/load
+      CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$
+      echo "delete cilium connectivity test resources and re-validate state"
+      kubectl delete ns cilium-test
+      kubectl get pod -owide -A
+      CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$
+    name: "validatePods"
+    displayName: "Validate Pods"
+
+  - script: |
+      echo "Run wireserver and metadata connectivity Tests"
+      bash test/network/wireserver_metadata_test.sh
+    retryCountOnTaskFailure: 3
+    name: "WireserverMetadataConnectivityTests"
+    displayName: "Run Wireserver and Metadata Connectivity Tests"
diff --git a/test/integration/manifests/cilium/cilium-config-dualstack.yaml b/test/integration/manifests/cilium/cilium-config-dualstack.yaml
@@ -0,0 +1,89 @@
+apiVersion: v1
+data:
+  agent-not-ready-taint-key: node.cilium.io/agent-not-ready
+  arping-refresh-period: 30s
+  auto-direct-node-routes: "false"
+  bpf-lb-external-clusterip: "false"
+  bpf-lb-map-max: "65536"
+  bpf-lb-mode: snat
+  bpf-map-dynamic-size-ratio: "0.0025"
+  bpf-policy-map-max: "16384"
+  bpf-root: /sys/fs/bpf
+  cgroup-root: /run/cilium/cgroupv2
+  cilium-endpoint-gc-interval: 5m0s
+  cluster-id: "0"
+  cluster-name: default
+  debug: "false"
+  disable-cnp-status-updates: "true"
+  disable-endpoint-crd: "false"
+  enable-auto-protect-node-port-range: "true"
+  enable-bgp-control-plane: "false"
+  enable-bpf-clock-probe: "true"
+  enable-endpoint-health-checking: "false"
+  enable-endpoint-routes: "true"
+  enable-health-check-nodeport: "true"
+  enable-health-checking: "true"
+  enable-host-legacy-routing: "true"
+  enable-hubble: "false"
+  enable-ipv4: "true"
+  enable-ipv4-masquerade: "false"
+  enable-ipv6: "true"
+  enable-ipv6-masquerade: "false"
+  enable-k8s-terminating-endpoint: "true"
+  enable-l2-neigh-discovery: "true"
+  enable-l7-proxy: "false"
+  enable-local-node-route: "false"
+  enable-local-redirect-policy: "false"
+  enable-metrics: "true"
+  enable-policy: default
+  enable-remote-node-identity: "true"
+  enable-session-affinity: "true"
+  enable-svc-source-range-check: "true"
+  enable-vtep: "false"
+  enable-well-known-identities: "false"
+  enable-xt-socket-fallback: "true"
+  identity-allocation-mode: crd
+  install-iptables-rules: "true"
+  install-no-conntrack-iptables-rules: "false"
+  ipam: delegated-plugin
+  kube-proxy-replacement: strict
+  kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256"
+  local-router-ipv4: 169.254.23.0
+  local-router-ipv6: "fe80::"
+  metrics: +cilium_bpf_map_pressure
+  monitor-aggregation: medium
+  monitor-aggregation-flags: all
+  monitor-aggregation-interval: 5s
+  node-port-bind-protection: "true"
+  nodes-gc-interval: 5m0s
+  operator-api-serve-addr: 127.0.0.1:9234
+  operator-prometheus-serve-addr: :9963
+  preallocate-bpf-maps: "false"
+  procfs: /host/proc
+  prometheus-serve-addr: :9962
+  remove-cilium-node-taints: "true"
+  set-cilium-is-up-condition: "true"
+  sidecar-istio-proxy-image: cilium/istio_proxy
+  synchronize-k8s-nodes: "true"
+  tofqdns-dns-reject-response-code: refused
+  tofqdns-enable-dns-compression: "true"
+  tofqdns-endpoint-max-ip-per-hostname: "50"
+  tofqdns-idle-connection-grace-period: 0s
+  tofqdns-max-deferred-connection-deletes: "10000"
+  tofqdns-min-ttl: "3600"
+  tofqdns-proxy-response-max-delay: 100ms
+  tunnel: disabled
+  unmanaged-pod-watcher-interval: "15"
+  vtep-cidr: ""
+  vtep-endpoint: ""
+  vtep-mac: ""
+  vtep-mask: ""
+kind: ConfigMap
+metadata:
+  annotations:
+    meta.helm.sh/release-name: cilium
+    meta.helm.sh/release-namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: cilium-config
+  namespace: kube-system