From fea43a3a75fca67841da99306778883a7a067231 Mon Sep 17 00:00:00 2001 From: Tahmid Alam Date: Thu, 8 Feb 2024 09:01:16 -0800 Subject: [PATCH] Cilium Load Test with Hubble Enabled. (#2558) * load test with huble enabled config file fix config file fix config file path fix fix: typo removing socket path to use default enabling hubble with parameters chaging the directory for cilium removing extra space setting version for cilium cleaning up exporting cilium image version replacing cilium hubble version var * addressed comments * changing daemonset dir to v1.14.4 * replaced variable with version --- .../cilium-overlay-load-test-template.yaml | 103 +++++++++++++----- .pipelines/cni/pipeline.yaml | 13 +++ .../cilium/cilium-config-hubble.yaml | 1 - 3 files changed, 86 insertions(+), 31 deletions(-) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 9325305102..1963ed3d65 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -8,6 +8,7 @@ parameters: os: "linux" arch: "" osSKU: Ubuntu + hubbleEnabled: false # Condition confirms that: # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success. @@ -48,38 +49,80 @@ stages: - setup displayName: "Cilium Test - ${{ parameters.name }}" jobs: - - job: deploy_cilium_components - steps: - - task: AzureCLI@1 - displayName: "Install Cilium, CNS, and ip-masq-agent" - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -ex - az extension add --name aks-preview - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - ls -lah - pwd - kubectl cluster-info - kubectl get po -owide -A + - ${{if eq(parameters.hubbleEnabled, false)}}: + - job: deploy_cilium_components + displayName: Deploy Cilium + steps: + - task: AzureCLI@1 + displayName: "Install Cilium, CNS, and ip-masq-agent" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml + + echo "install Cilium onto Overlay Cluster" + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + + + echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + kubectl get po -owide -A + + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + kubectl get po -owide -A + - ${{if eq(parameters.hubbleEnabled, true)}}: + - job: deploy_cilium_components + displayName: Deploy Cilium with Hubble + steps: + - task: AzureCLI@1 + displayName: "Install Cilium, CNS, and ip-masq-agent" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A - echo "deploy Cilium ConfigMap" - kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml - echo "install Cilium onto Overlay Cluster" - kubectl apply -f test/integration/manifests/cilium/cilium-agent - kubectl apply -f test/integration/manifests/cilium/cilium-operator - echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster" - # Passes Cilium image to daemonset and deployment - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - - kubectl get po -owide -A + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-config/cilium-config-hubble.yaml + + echo "install Cilium onto Overlay Cluster with hubble enabled" + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-agent/files + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-operator/files + + echo "install Cilium v1.14.4 onto Overlay Cluster" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl | kubectl apply -f - + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl | kubectl apply -f - + kubectl get po -owide -A - echo "Deploy Azure-CNS" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) - kubectl get po -owide -A + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + kubectl get po -owide -A + - job: deploy_pods condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Scale Test" diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index afa1cc128d..f917b9bae3 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -162,6 +162,15 @@ stages: nodeCount: ${NODE_COUNT_CILIUM} vmSize: ${VM_SIZE_CILIUM} + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_hubble + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-hub" + hubbleEnabled: true + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + - template: cilium/cilium-overlay-load-test-template.yaml parameters: name: cilium_overlay_mariner @@ -269,6 +278,7 @@ stages: condition: always() dependsOn: - cilium_overlay + - cilium_overlay_hubble - cilium_overlay_mariner - cilium_overlay_arm - cilium_overlay_rdma @@ -292,6 +302,9 @@ stages: cilium_overlay: name: cilium_overlay clusterName: "cilium-over" + cilium_overlay_hubble: + name: cilium_overlay_hubble + clusterName: "cil-over-hub" cilium_overlay_mariner: name: cilium_overlay_mariner clusterName: "cil-over-mar" diff --git a/test/integration/manifests/cilium/cilium-config-hubble.yaml b/test/integration/manifests/cilium/cilium-config-hubble.yaml index 492fc08bfc..63115ae5ff 100644 --- a/test/integration/manifests/cilium/cilium-config-hubble.yaml +++ b/test/integration/manifests/cilium/cilium-config-hubble.yaml @@ -49,7 +49,6 @@ data: hubble-metrics-server: :9965 hubble-disable-tls: "false" hubble-listen-address: "" - hubble-socket-path: /dev/null hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key