diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml index 2ebc576077..014adfde0a 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml @@ -119,7 +119,7 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if eq(parameters.dualstack, true) }}: + - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}: - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: DualStack Test @@ -130,3 +130,14 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 + - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}: + - template: ../k8s-e2e/k8s-e2e-step-template.yaml + parameters: + testName: DualStack Test|Cilium + name: DualStack + clusterName: ${{ parameters.clusterName }} + ginkgoFocus: '\[Feature:IPv6DualStack\]' + ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135 + os: ${{ parameters.os }} + processes: 8 + attempts: 3 diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 460ebd0384..d58d867fa2 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -420,6 +420,18 @@ stages: k8sVersion: "" dependsOn: "test" + # Cilium Dualstack Overlay E2E tests + - template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml + parameters: + name: "cilium_dualstackoverlay_e2e" + displayName: Cilium on AKS DualStack Overlay + os: linux + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cildsovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "test" + # Cilium Overlay with hubble E2E tests - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml parameters: @@ -517,6 +529,7 @@ stages: - aks_swift_vnetscale_e2e - aks_windows_22_e2e - dualstackoverlay_e2e + - cilium_dualstackoverlay_e2e variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: @@ -562,6 +575,9 @@ stages: dualstackoverlay_e2e: name: dualstackoverlay_e2e clusterName: "dsovere2e" + cilium_dualstackoverlay_e2e: + name: cilium_dualstackoverlay_e2e + clusterName: "cildsovere2e" steps: - template: templates/delete-cluster.yaml parameters: @@ -600,6 +616,7 @@ stages: - aks_ubuntu_22_linux_e2e - aks_windows_22_e2e - dualstackoverlay_e2e + - cilium_dualstackoverlay_e2e jobs: - job: delete_remote_artifacts displayName: Delete remote artifacts diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml new file mode 100644 index 0000000000..bf8cb35b2a --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml @@ -0,0 +1,82 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cilium Dualstack Overlay Test Suite - (${{ parameters.name }}) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: cilium-dualstackoverlay-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + dependsOn: ${{ parameters.name }} + dualstack: true + dns: true + portforward: true + service: true + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml new file mode 100644 index 0000000000..e91282f6d4 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -0,0 +1,124 @@ +parameters: + name: "" + clusterName: "" + +steps: + + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/cilium-config-dualstack.yaml + echo "install Cilium ${CILIUM_VERSION_TAG}" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + # Use different file directories for nightly and current cilium version + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + kubectl get po -owide -A + name: "installCilium" + displayName: "Install Cilium on AKS Dualstack Overlay" + + - script: | + echo "install cilium CLI" + if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then + echo "Cilium Agent Version ${BASH_REMATCH[0]}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt) + else + echo "Cilium Agent Version ${CILIUM_VERSION_TAG}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) + fi + CLI_ARCH=amd64 + if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum + sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin + rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + cilium status + cilium version + name: "installCiliumCLI" + displayName: "Install Cilium CLI" + + - script: | + echo "Start Azilium E2E Tests on Overlay Cluster" + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true + retryCountOnTaskFailure: 3 + name: "aziliumTest" + displayName: "Run Azilium E2E on AKS Overlay" + + - script: | + kubectl get pods -A + echo "Waiting < 2 minutes for cilium to be ready" + # Ensure Cilium is ready Xm\Xs + cilium status --wait --wait-duration 2m + retryCountOnTaskFailure: 3 + name: "CiliumStatus" + displayName: "Cilium Status" + + - script: | + echo "Run Cilium Connectivity Tests" + cilium status + cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' + retryCountOnTaskFailure: 3 + name: "ciliumConnectivityTests" + displayName: "Run Cilium Connectivity Tests" + + - script: | + set -e + kubectl get po -owide -A + cd test/integration/datapath + echo "Dualstack Overlay Linux datapath IPv6 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true + echo "Dualstack Overlay Linux datapath IPv4 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + retryCountOnTaskFailure: 3 + name: "DualStack_Overlay_Linux_Tests" + displayName: "DualStack Overlay Linux Tests" + + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl get pod -owide -A + # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. + # Saves 17 minutes + kubectl delete deploy -n cilium-test echo-external-node + cd test/integration/load + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns cilium-test + kubectl get pod -owide -A + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + name: "validatePods" + displayName: "Validate Pods" + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" diff --git a/test/integration/manifests/cilium/cilium-config-dualstack.yaml b/test/integration/manifests/cilium/cilium-config-dualstack.yaml new file mode 100644 index 0000000000..7ca869e666 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-config-dualstack.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: default + debug: "false" + disable-cnp-status-updates: "true" + disable-endpoint-crd: "false" + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-routes: "true" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "true" + enable-hubble: "false" + enable-ipv4: "true" + enable-ipv4-masquerade: "false" + enable-ipv6: "true" + enable-ipv6-masquerade: "false" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "false" + enable-metrics: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + kube-proxy-replacement: strict + kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256" + local-router-ipv4: 169.254.23.0 + local-router-ipv6: "fe80::" + metrics: +cilium_bpf_map_pressure + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + node-port-bind-protection: "true" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: 100ms + tunnel: disabled + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index 2e2dcc807d..4c12cb7833 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -46,6 +46,11 @@ var linuxChecksMap = map[string][]check{ {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, {"azure dualstackoverlay", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, }, + "cilium_dualstack": { + {"cns dualstack", cnsManagedStateFileDualStackIps, cnsLabelSelector, privilegedNamespace, cnsManagedStateFileCmd}, // cns configmap "ManageEndpointState": true, | Endpoints managed in CNS State File + {"cilium", ciliumStateFileDualStackIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, + }, } type CnsManagedState struct { @@ -70,7 +75,8 @@ type NetworkingAddressing struct { } type Address struct { - Addr string `json:"ipv4"` + IPv4 string `json:"ipv4"` + IPv6 string `json:"ipv6"` } // parse azure-vnet.json @@ -133,8 +139,26 @@ func cnsManagedStateFileIps(result []byte) (map[string]string, error) { for _, v := range cnsResult.Endpoints { for ifName, ip := range v.IfnameToIPMap { if ifName == "eth0" { - ip := ip.IPv4[0].IP.String() - cnsPodIps[ip] = v.PodName + cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName + } + } + } + return cnsPodIps, nil +} + +func cnsManagedStateFileDualStackIps(result []byte) (map[string]string, error) { + var cnsResult CnsManagedState + err := json.Unmarshal(result, &cnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list") + } + + cnsPodIps := make(map[string]string) + for _, v := range cnsResult.Endpoints { + for ifName, ip := range v.IfnameToIPMap { + if ifName == "eth0" { + cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName + cnsPodIps[ip.IPv6[0].IP.String()] = v.PodName } } } @@ -151,8 +175,27 @@ func ciliumStateFileIps(result []byte) (map[string]string, error) { ciliumPodIps := make(map[string]string) for _, v := range ciliumResult { for _, addr := range v.Status.Networking.Addresses { - if addr.Addr != "" { - ciliumPodIps[addr.Addr] = v.Status.Networking.InterfaceName + if addr.IPv4 != "" { + ciliumPodIps[addr.IPv4] = v.Status.Networking.InterfaceName + } + } + } + return ciliumPodIps, nil +} + +func ciliumStateFileDualStackIps(result []byte) (map[string]string, error) { + var ciliumResult []CiliumEndpointStatus + err := json.Unmarshal(result, &ciliumResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cilium endpoint list") + } + + ciliumPodIps := make(map[string]string) + for _, v := range ciliumResult { + for _, addr := range v.Status.Networking.Addresses { + if addr.IPv4 != "" && addr.IPv6 != "" { + ciliumPodIps[addr.IPv4] = v.Status.Networking.InterfaceName + ciliumPodIps[addr.IPv6] = v.Status.Networking.InterfaceName } } }