From 89b98ddbc12a13d2d9d782112c77e53b4caf0639 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Mon, 27 Nov 2023 16:05:41 -0800 Subject: [PATCH 01/57] changed to run nightly build pipeline only on sunday at 6 AM Signed-off-by: vivekr-splunk --- .github/workflows/nightly-int-test-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 91537feac..065743958 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -1,7 +1,7 @@ name: Nightly Integration Test WorkFlow on: schedule: - - cron: "0 06 * * *" + - cron: "0 06 * * 0" jobs: build-operator-image: runs-on: ubuntu-latest From 11d92a400dae1fad8024cf886d6402dc8613eec5 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Wed, 6 Dec 2023 11:38:41 -0800 Subject: [PATCH 02/57] testing graviton cluster creation and pipeline testing Signed-off-by: vivekr-splunk --- .github/workflows/build-test-push-workflow.yml | 1 + test/deploy-eks-cluster.sh | 11 ++++++++++- test/deploy-kind-cluster.sh | 4 ++-- test/env.sh | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index f87fc8635..411d11d1b 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -153,6 +153,7 @@ jobs: # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" TEST_CLUSTER_PLATFORM: eks + EKS_INSTANCE_TYPE: m6g.large EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index a4e7d47f0..1018719a2 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -20,6 +20,15 @@ if [[ -z "${EKS_CLUSTER_K8_VERSION}" ]]; then export EKS_CLUSTER_K8_VERSION="1.26" fi +if [[ -z "${EKS_CLUSTER_INSTANCE_TYPE}" ]]; then + echo "EKS_INSTANCE_TYPE not set. Chaning to env.sh value" + export EKS_CLUSTER_INSTANCE_TYPE = "${EKS_INSTANCE_TYPE}" +else + export EKS_CLUSTER_INSTANCE_TYPE = "m5.2xlarge" +fi + + + function deleteCluster() { echo "Cleanup remaining PVC on the EKS Cluster ${TEST_CLUSTER_NAME}" tools/cleanup.sh @@ -57,7 +66,7 @@ function createCluster() { found=$(eksctl get cluster --name "${TEST_CLUSTER_NAME}" -v 0) if [ -z "${found}" ]; then - eksctl create cluster --name=${TEST_CLUSTER_NAME} --nodes=${CLUSTER_WORKERS} --vpc-public-subnets=${EKS_VPC_PUBLIC_SUBNET_STRING} --vpc-private-subnets=${EKS_VPC_PRIVATE_SUBNET_STRING} --instance-types=m5.2xlarge --version=${EKS_CLUSTER_K8_VERSION} + eksctl create cluster --name=${TEST_CLUSTER_NAME} --nodes=${CLUSTER_WORKERS} --vpc-public-subnets=${EKS_VPC_PUBLIC_SUBNET_STRING} --vpc-private-subnets=${EKS_VPC_PRIVATE_SUBNET_STRING} --instance-types=${EKS_CLUSTER_INSTANCE_TYPE} --version=${EKS_CLUSTER_K8_VERSION} if [ $? -ne 0 ]; then echo "Unable to create cluster - ${TEST_CLUSTER_NAME}" return 1 diff --git a/test/deploy-kind-cluster.sh b/test/deploy-kind-cluster.sh index d63598355..e704ea16d 100755 --- a/test/deploy-kind-cluster.sh +++ b/test/deploy-kind-cluster.sh @@ -10,7 +10,7 @@ function deleteCluster() { return 1 fi - docker rm -f ${reg_name} + docker rm -f ${reg_name} if [ $? -ne 0 ]; then echo "Unable to delete private registry - ${reg_name}" return 1 @@ -41,7 +41,7 @@ function createCluster() { workerNodes="- role: worker" for i in $(seq 2 $NUM_WORKERS);do workerNodes="${workerNodes}"$'\n'"- role: worker" - done + done # create a cluster with the local registry enabled in containerd cat < Date: Mon, 11 Dec 2023 10:30:44 -0800 Subject: [PATCH 03/57] supporting multi stage build for different platform Signed-off-by: vivekr-splunk --- Dockerfile | 11 ++++++++--- Makefile | 12 +++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a323ef4a..91282dd98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ # Build the manager binary -FROM golang:1.21.1 as builder +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.21.1 as builder + +ARG TARGETPLATFORM +ARG BUILDPLATFORM +ARG TARGETOS +ARG TARGETARCH WORKDIR /workspace # Copy the Go Modules manifests @@ -18,11 +23,11 @@ COPY tools/ tools/ COPY hack hack/ # Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o manager main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM registry.access.redhat.com/ubi8/ubi:latest +FROM --platform=${TARGETPLATFORM:-linux/amd64} registry.access.redhat.com/ubi8/ubi:latest ENV OPERATOR=/manager \ USER_UID=1001 \ diff --git a/Makefile b/Makefile index dd59513ae..17d1515de 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ VERSION ?= 2.2.1 SPLUNK_ENTERPRISE_IMAGE ?= "docker.io/splunk/splunk:edge" # WATCH_NAMESPACE defines if its clusterwide operator or namespace specific -# by default we leave it as clusterwide if it has to be namespace specific, +# by default we leave it as clusterwide if it has to be namespace specific, # add namespace to this WATCH_NAMESPACE ?= "" -# NAMESPACE defines default namespace where operator will be installed +# NAMESPACE defines default namespace where operator will be installed NAMESPACE ?= "splunk-operator" # CHANNELS define the bundle channels used in the bundle. @@ -25,6 +25,8 @@ ifneq ($(origin CHANNELS), undefined) BUNDLE_CHANNELS := --channels=$(CHANNELS) endif +export DOCKER_CLI_EXPERIMENTAL=enabled + # DEFAULT_CHANNEL defines the default channel used in the bundle. # Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") # To re-generate a bundle for any other default channel without changing the default setup, you can: @@ -153,12 +155,12 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx docker-buildx: test ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + # sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - docker buildx create --name project-v3-builder docker buildx use project-v3-builder - - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross + - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} . - docker buildx rm project-v3-builder - rm Dockerfile.cross + # rm Dockerfile.cross ##@ Deployment From a7638faf01366fd4dc2953d78864935380e89cad Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Tue, 16 Jan 2024 13:47:58 -0800 Subject: [PATCH 04/57] adding more time for test --- test/run-tests.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index 27e01ac29..6eabde8bc 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -22,7 +22,7 @@ if [ -n "${PRIVATE_REGISTRY}" ]; then PRIVATE_SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_REGISTRY}/${SPLUNK_ENTERPRISE_IMAGE} echo "docker images -q ${SPLUNK_OPERATOR_IMAGE}" # Don't pull splunk operator if exists locally since we maybe building it locally - if [ -z $(docker images -q ${SPLUNK_OPERATOR_IMAGE}) ]; then + if [ -z $(docker images -q ${SPLUNK_OPERATOR_IMAGE}) ]; then docker pull ${SPLUNK_OPERATOR_IMAGE} if [ $? -ne 0 ]; then echo "Unable to pull ${SPLUNK_OPERATOR_IMAGE}. Exiting..." @@ -55,7 +55,7 @@ if [ -n "${PRIVATE_REGISTRY}" ]; then docker images fi -if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then +if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then echo "Installing Splunk Operator using Helm charts" helm uninstall splunk-operator -n splunk-operator if [ "${CLUSTER_WIDE}" != "true" ]; then @@ -63,10 +63,10 @@ if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then else helm install splunk-operator --create-namespace --namespace splunk-operator --set splunkOperator.image.repository=${PRIVATE_SPLUNK_OPERATOR_IMAGE} --set image.repository=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} helm-chart/splunk-operator fi -elif [ "${CLUSTER_WIDE}" != "true" ]; then +elif [ "${CLUSTER_WIDE}" != "true" ]; then # Install the CRDs echo "Installing enterprise CRDs..." - make kustomize + make kustomize make uninstall bin/kustomize build config/crd | kubectl create -f - else @@ -79,7 +79,7 @@ if [ $? -ne 0 ]; then exit 1 fi -if [ "${CLUSTER_WIDE}" == "true" ]; then +if [ "${CLUSTER_WIDE}" == "true" ]; then echo "wait for operator pod to be ready..." # sleep before checking for deployment, in slow clusters deployment call may not even started # in those cases, kubectl will fail with error: no matching resources found @@ -98,14 +98,14 @@ if [ -z "$rc" ]; then go get github.com/onsi/gomega/... go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@latest -fi +fi echo "Running test using number of nodes: ${NUM_NODES}" echo "Running test using these images: ${PRIVATE_SPLUNK_OPERATOR_IMAGE} and ${PRIVATE_SPLUNK_ENTERPRISE_IMAGE}..." -# Check if test focus is set +# Check if test focus is set if [[ -z "${TEST_FOCUS}" ]]; then TEST_TO_RUN="${TEST_REGEX}" echo "Test focus not set running smoke test by default :: ${TEST_TO_RUN}" @@ -191,5 +191,5 @@ fi echo "Skipping following test :: ${TEST_TO_SKIP}" # Running only smoke test cases by default or value passed through TEST_FOCUS env variable. To run different test packages add/remove path from focus argument or TEST_FOCUS variable -echo "ginkgo --junit-report=inttest.xml -vv --keep-going --trace -r --timeout=3h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE}" -ginkgo --junit-report=inttest-junit.xml --output-dir=`pwd` -vv --keep-going --trace -r --timeout=3h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE} \ No newline at end of file +echo "ginkgo --junit-report=inttest.xml -vv --keep-going --trace -r --timeout=7h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE}" +ginkgo --junit-report=inttest-junit.xml --output-dir=`pwd` -vv --keep-going --trace -r --timeout=7h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE} \ No newline at end of file From 0e46347682f70b7d679ccd0f75cf1d915b3c255a Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Tue, 23 Jan 2024 12:23:06 -0800 Subject: [PATCH 05/57] changing cpu type --- .github/workflows/build-test-push-workflow.yml | 2 +- .github/workflows/int-test-workflow.yml | 1 + .github/workflows/manual-int-test-workflow.yml | 1 + .github/workflows/nightly-int-test-workflow.yml | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index 411d11d1b..c8d1ed136 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -153,7 +153,7 @@ jobs: # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" TEST_CLUSTER_PLATFORM: eks - EKS_INSTANCE_TYPE: m6g.large + EKS_INSTANCE_TYPE: m5.2xlarge EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 04432c084..be1d43383 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -74,6 +74,7 @@ jobs: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator + EKS_INSTANCE_TYPE: m5.2xlarge TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 0aad051ed..bed26eef5 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -40,6 +40,7 @@ jobs: SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator TEST_FOCUS: "${{ matrix.test }}" + EKS_INSTANCE_TYPE: m5.2xlarge # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" TEST_CLUSTER_PLATFORM: eks diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 065743958..92dfae436 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -80,6 +80,7 @@ jobs: SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator TEST_FOCUS: "${{ matrix.test }}" + EKS_INSTANCE_TYPE: m5.2xlarge # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" TEST_CLUSTER_PLATFORM: eks From 78fe2ed91927433bf9e29967041c3ce151343b31 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 12:48:29 -0800 Subject: [PATCH 06/57] graviton build and test --- .../graviton-build-test-push-workflow.yml | 329 ++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 .github/workflows/graviton-build-test-push-workflow.yml diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml new file mode 100644 index 000000000..8ea3b61d9 --- /dev/null +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -0,0 +1,329 @@ +name: Build and Test +on: push +jobs: + check-formating: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Check Source formatting + run: make fmt && if [[ $? -ne 0 ]]; then false; fi + - name: Lint source code + run: make vet && if [[ $? -ne 0 ]]; then false; fi + unit-tests: + runs-on: ubuntu-latest + needs: check-formating + steps: + - uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Install goveralls + run: | + go version + go install github.com/mattn/goveralls@latest + - name: Install Ginkgo + run: | + make setup/ginkgo + go mod tidy + - name: Run Unit Tests + run: make test + - name: Run Code Coverage + run: goveralls -coverprofile=coverage.out -service=circle-ci -repotoken ${{ secrets.COVERALLS_TOKEN }} + - name: Upload Coverage artifacts + uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 + with: + name: coverage.out + path: coverage.out + build-operator-image: + runs-on: ubuntu-latest + needs: unit-tests + env: + SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Install Ginkgo + run: | + make setup/ginkgo + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Install Operator SDK + run: | + export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) + export OS=$(uname | awk '{print tolower($0)}') + export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }} + sudo curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} + sudo chmod +x operator-sdk_${OS}_${ARCH} + sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Make Splunk Operator Image + run: | + make docker-buildx IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 + - name: Push Splunk Operator Image to ECR + run: | + echo "Uploading Image to ECR:: ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA" + make docker-push IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + vulnerability-scan: + runs-on: ubuntu-latest + needs: build-operator-image + env: + SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v1 + - name: Pull Splunk Operator Image Locally + run: | + docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Setup clair scanner + run: make setup_clair_scanner + - name: Scan container image + run: make run_clair_scan IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Stop clair scanner + run: make stop_clair_scanner + - name: Save scan results as artifacts + uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 + with: + name: clair-scanner-logs + path: clair-scanner-logs + smoke-tests: + needs: vulnerability-scan + strategy: + fail-fast: false + matrix: + test: [ + basic, + appframeworks1, + managerappframeworkc3, + managerappframeworkm4, + managersecret, + managermc, + ] + runs-on: ubuntu-latest + env: + CLUSTER_NODES: 1 + CLUSTER_WORKERS: 3 + SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} + SPLUNK_ENTERPRISE_RELEASE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator + TEST_FOCUS: "${{ matrix.test }}" + # This regex matches any string not containing smoke keyword + TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" + TEST_CLUSTER_PLATFORM: eks + EKS_INSTANCE_TYPE: m5.2xlarge + EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} + EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} + TEST_BUCKET: ${{ secrets.TEST_BUCKET }} + TEST_INDEXES_S3_BUCKET: ${{ secrets.TEST_INDEXES_S3_BUCKET }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + PRIVATE_REGISTRY: ${{ secrets.ECR_REPOSITORY }} + S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ENTERPRISE_LICENSE_LOCATION: ${{ secrets.ENTERPRISE_LICENSE_LOCATION }} + EKS_SSH_PUBLIC_KEY: ${{ secrets.EKS_SSH_PUBLIC_KEY }} + CLUSTER_WIDE: "true" + DEPLOYMENT_TYPE: "" + steps: + - name: Set Test Cluster Name + run: | + echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV + - name: Chekcout code + uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Change splunk enterprise to release image on main branches + if: github.ref == 'refs/heads/main' + run: | + echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV + - name: Install Kubectl + uses: Azure/setup-kubectl@v3 + with: + version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} + - name: Install Python + uses: actions/setup-python@v2 + - name: Install AWS CLI + run: | + curl "${{ steps.dotenv.outputs.AWSCLI_URL}}" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update + aws --version + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Install Ginkgo + run: | + make setup/ginkgo + - name: Install Helm + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + DESIRED_VERSION=v3.8.2 bash get_helm.sh + - name: Install EKS CTL + run: | + curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp + sudo mv /tmp/eksctl /usr/local/bin + eksctl version + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Install Operator SDK + run: | + sudo curl -L -o /usr/local/bin/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}/operator-sdk-${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}-x86_64-linux-gnu + sudo chmod +x /usr/local/bin/operator-sdk + - name: Configure Docker Hub credentials + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN}} + - name: Pull Splunk Enterprise Image + run: docker pull ${{ env.SPLUNK_ENTERPRISE_IMAGE }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Tag and Push Splunk Enterprise Image to ECR + run: | + docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + docker push ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + - name: Pull Splunk Operator Image Locally + run: | + docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Change Operator Image Tag to latest + run: | + docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:latest + - name: Create EKS cluster + run: | + export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} + export PLATFORMS=linux/arm64 + export BUILDPLATFORM=linux/arm64 + export TARGETOS=linux + export TARGETARCH=arm64 + export EKS_INSTANCE_TYPE=c6g.4xlarge + make cluster-up + - name: install metric server + run: | + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + - name: install k8s dashboard + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml + - name: Setup Kustomize + run: | + sudo snap install kustomize + mkdir -p ./bin + cp /snap/bin/kustomize ./bin/kustomize + - name: Run smoke test + id: smoketest + run: | + make int-test + - name: Collect Test Logs + if: ${{ always() }} + run: | + mkdir -p /tmp/pod_logs + find ./test -name "*.log" -exec cp {} /tmp/pod_logs \; + - name: Archive Pod Logs + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: "splunk-pods-logs--artifacts-${{ matrix.test }}" + path: "/tmp/pod_logs/**" + - name: Cleanup Test Case artifacts + if: ${{ always() }} + run: | + make cleanup + make clean + - name: Cleanup up EKS cluster + if: ${{ always() }} + run: | + make cluster-down + #- name: Test Report + # uses: dorny/test-reporter@v1 + # if: success() || failure() # run this step even if previous step failed + # with: + # name: Integration Tests # Name of the check run which will be created + # path: inttest-*.xml # Path to test results + # reporter: jest-junit # Format of test results + push-latest: + needs: smoke-tests + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + env: + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + TAG: latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Configure Docker Hub credentials + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PUSH_TOKEN}} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v1 + - name: Pull Splunk Operator Image Locally + run: | + docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Change Operator Image Tag to latest + run: | + docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:${{ env.TAG }} + - name: Push Splunk Operator Image to Docker Hub + run: docker push ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:${{ env.TAG }} \ No newline at end of file From 26204c9e46fc2b062af78b4267d79a86d0434f59 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 12:49:06 -0800 Subject: [PATCH 07/57] graviton build and test --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 8ea3b61d9..aed8b8078 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -1,4 +1,4 @@ -name: Build and Test +name: Graviton Build and Test on: push jobs: check-formating: From 3892c1d1300760461c6ce53669d7c77110174f51 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 15:20:12 -0800 Subject: [PATCH 08/57] adding one command to build and push docker image --- .github/workflows/graviton-build-test-push-workflow.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index aed8b8078..edfd1b5de 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -84,13 +84,9 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - - name: Make Splunk Operator Image + - name: Make Splunk Operator Image and Push to ECR run: | - make docker-buildx IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 - - name: Push Splunk Operator Image to ECR - run: | - echo "Uploading Image to ECR:: ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA" - make docker-push IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + make docker-buildx docker-push IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 vulnerability-scan: runs-on: ubuntu-latest needs: build-operator-image From 3346fb21c54c1ce540e5ebeb9bb12b3fd85366b4 Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Thu, 22 Feb 2024 18:24:30 -0600 Subject: [PATCH 09/57] Remove push as docker buildx uses --push option already --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index edfd1b5de..1571bfd7c 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -86,7 +86,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push to ECR run: | - make docker-buildx docker-push IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 + make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 vulnerability-scan: runs-on: ubuntu-latest needs: build-operator-image From ab6c7b24ff53ce26efef8d9b66548c58480f42d8 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 17:02:47 -0800 Subject: [PATCH 10/57] changes done to fix eks creation Signed-off-by: vivekr-splunk --- test/deploy-eks-cluster.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index 1018719a2..2d41ecc98 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -22,9 +22,9 @@ fi if [[ -z "${EKS_CLUSTER_INSTANCE_TYPE}" ]]; then echo "EKS_INSTANCE_TYPE not set. Chaning to env.sh value" - export EKS_CLUSTER_INSTANCE_TYPE = "${EKS_INSTANCE_TYPE}" + export EKS_CLUSTER_INSTANCE_TYPE="${EKS_INSTANCE_TYPE}" else - export EKS_CLUSTER_INSTANCE_TYPE = "m5.2xlarge" + export EKS_CLUSTER_INSTANCE_TYPE="m5.2xlarge" fi From 3ce03509e2d0cb3f426fe140d2001a4b48225594 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 17:05:23 -0800 Subject: [PATCH 11/57] adding one command to build and push docker image --- .env | 4 ++-- config/manager/kustomization.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 12038fdfc..ee3e4095f 100644 --- a/.env +++ b/.env @@ -5,5 +5,5 @@ AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 -EKS_CLUSTER_K8_VERSION=1.27 -SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.3 \ No newline at end of file +EKS_CLUSTER_K8_VERSION=1.28 +SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.3 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 8e92ce43b..4178c194a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -16,5 +16,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: docker.io/splunk/splunk-operator - newTag: 2.5.1 + newName: docker.io/vivekrsplunk/splunk-operator + newTag: 2.5.2 From 5dcdbffce6927879a6563e0b4dd7582d37511c66 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 18:16:51 -0800 Subject: [PATCH 12/57] removed vulnerability scan --- .../graviton-build-test-push-workflow.yml | 41 +------------------ 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 1571bfd7c..0fbd4d7e2 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -86,46 +86,9 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push to ECR run: | - make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 - vulnerability-scan: - runs-on: ubuntu-latest - needs: build-operator-image - env: - SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} - SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator - ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - steps: - - uses: actions/checkout@v2 - - name: Dotenv Action - id: dotenv - uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.5.0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v1 - - name: Pull Splunk Operator Image Locally - run: | - docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - - name: Setup clair scanner - run: make setup_clair_scanner - - name: Scan container image - run: make run_clair_scan IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - - name: Stop clair scanner - run: make stop_clair_scanner - - name: Save scan results as artifacts - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 - with: - name: clair-scanner-logs - path: clair-scanner-logs + make docker-buildx docker-push IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 smoke-tests: - needs: vulnerability-scan + needs: build-operator-image strategy: fail-fast: false matrix: From 515b2efbe4fa8a9e7e380a06a2d1df15f22ce9ed Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 20:42:38 -0800 Subject: [PATCH 13/57] fixed an issue buildx --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 0fbd4d7e2..79ca28b8a 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -86,7 +86,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push to ECR run: | - make docker-buildx docker-push IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 + make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 smoke-tests: needs: build-operator-image strategy: From 74b44e7fb2006173fda156c092a5fd9439896023 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 21:48:07 -0800 Subject: [PATCH 14/57] testing to add buildx for arm64 --- .../workflows/graviton-build-test-push-workflow.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 79ca28b8a..7a49142bb 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -94,11 +94,11 @@ jobs: matrix: test: [ basic, - appframeworks1, - managerappframeworkc3, - managerappframeworkm4, - managersecret, - managermc, + #appframeworks1, + #managerappframeworkc3, + #managerappframeworkm4, + #managersecret, + #managermc, ] runs-on: ubuntu-latest env: @@ -265,6 +265,8 @@ jobs: uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.5.0 + with: + platforms: "linux/arm64" - name: Configure Docker Hub credentials uses: docker/login-action@v1 with: From 0821a936be4e65f056bb358e158acc3e7d72dfe8 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:13:11 -0800 Subject: [PATCH 15/57] adding macos14 as its arm64 --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 7a49142bb..c201e7baf 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -253,7 +253,7 @@ jobs: push-latest: needs: smoke-tests if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest + runs-on: macos-14 env: SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator TAG: latest From 2cef6d4b6ff11cd11f9f096061af12a75d5ff1e3 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:16:09 -0800 Subject: [PATCH 16/57] changing everything to macos-14 --- .github/workflows/graviton-build-test-push-workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index c201e7baf..b1d380c6d 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -2,7 +2,7 @@ name: Graviton Build and Test on: push jobs: check-formating: - runs-on: ubuntu-latest + runs-on: macos-14 steps: - uses: actions/checkout@v2 - name: Dotenv Action @@ -17,7 +17,7 @@ jobs: - name: Lint source code run: make vet && if [[ $? -ne 0 ]]; then false; fi unit-tests: - runs-on: ubuntu-latest + runs-on: macos-14 needs: check-formating steps: - uses: actions/checkout@v2 @@ -46,7 +46,7 @@ jobs: name: coverage.out path: coverage.out build-operator-image: - runs-on: ubuntu-latest + runs-on: macos-14 needs: unit-tests env: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} @@ -100,7 +100,7 @@ jobs: #managersecret, #managermc, ] - runs-on: ubuntu-latest + runs-on: macos-14 env: CLUSTER_NODES: 1 CLUSTER_WORKERS: 3 From a0710819c8156aca9866cb4df614fc2f71746d58 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:32:31 -0800 Subject: [PATCH 17/57] adding mac operator-sdk download --- .github/workflows/graviton-build-test-push-workflow.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index b1d380c6d..140751fa2 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -171,7 +171,10 @@ jobs: uses: docker/setup-buildx-action@v2.5.0 - name: Install Operator SDK run: | - sudo curl -L -o /usr/local/bin/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}/operator-sdk-${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}-x86_64-linux-gnu + export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) + export OS=$(uname | awk '{print tolower($0)}') + export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.33.0 + sudo curl -L -o /usr/local/bin/operator-sdk ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} sudo chmod +x /usr/local/bin/operator-sdk - name: Configure Docker Hub credentials uses: docker/login-action@v1 From 37ca51484430e9bb43f298e7d6036b7699031ea7 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:34:47 -0800 Subject: [PATCH 18/57] changed back eks version 1.27 --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index ee3e4095f..d79c25287 100644 --- a/.env +++ b/.env @@ -5,5 +5,5 @@ AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 -EKS_CLUSTER_K8_VERSION=1.28 +EKS_CLUSTER_K8_VERSION=1.27 SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.3 From 9ea1e42948d26082aeb6bd1b07cef83d0a4784ca Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:39:42 -0800 Subject: [PATCH 19/57] splunk arm64 9.2.0 build --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index d79c25287..4564213fa 100644 --- a/.env +++ b/.env @@ -6,4 +6,4 @@ KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 EKS_CLUSTER_K8_VERSION=1.27 -SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.3 +SPLUNK_ENTERPRISE_RELEASE_IMAGE=vivekrsplunk/splunk-aarch64:9.2.0 From a828a257591688ac8ceed0080273b33d31e246af Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 22:47:29 -0800 Subject: [PATCH 20/57] changing build to be on ubuntu --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 140751fa2..5198bbc79 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -46,7 +46,7 @@ jobs: name: coverage.out path: coverage.out build-operator-image: - runs-on: macos-14 + runs-on: ubuntu-latest needs: unit-tests env: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} From 6f43d941bf19dfc28a4dd9fdb0b7a96b8e5ff4d4 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 23:08:56 -0800 Subject: [PATCH 21/57] awscli arm build --- .env | 1 + .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 4564213fa..32bab5459 100644 --- a/.env +++ b/.env @@ -2,6 +2,7 @@ OPERATOR_SDK_VERSION=v1.31.0 REVIEWERS=vivekr-splunk,akondur GO_VERSION=1.21.5 AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip +ARMCLI_ARM_URL=https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 5198bbc79..7f20e1a9c 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -145,7 +145,7 @@ jobs: uses: actions/setup-python@v2 - name: Install AWS CLI run: | - curl "${{ steps.dotenv.outputs.AWSCLI_URL}}" -o "awscliv2.zip" + curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL}}" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install --update aws --version From 2e1d29003acfd12199fb442806233d556272b2b4 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 23:11:57 -0800 Subject: [PATCH 22/57] eksctl arm build --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 7f20e1a9c..b8a8cc481 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -164,7 +164,7 @@ jobs: DESIRED_VERSION=v3.8.2 bash get_helm.sh - name: Install EKS CTL run: | - curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp + curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_arm64.tar.gz" | tar xz -C /tmp sudo mv /tmp/eksctl /usr/local/bin eksctl version - name: Set up Docker Buildx From a0df39f1bdabdffa73842631f17c262095b20a53 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 23:14:09 -0800 Subject: [PATCH 23/57] disable unit test for now --- .../graviton-build-test-push-workflow.yml | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index b8a8cc481..815e4d5ce 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -16,38 +16,38 @@ jobs: run: make fmt && if [[ $? -ne 0 ]]; then false; fi - name: Lint source code run: make vet && if [[ $? -ne 0 ]]; then false; fi - unit-tests: - runs-on: macos-14 - needs: check-formating - steps: - - uses: actions/checkout@v2 - - name: Dotenv Action - id: dotenv - uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 - - name: Setup Go - uses: actions/setup-go@v2 - with: - go-version: ${{ steps.dotenv.outputs.GO_VERSION }} - - name: Install goveralls - run: | - go version - go install github.com/mattn/goveralls@latest - - name: Install Ginkgo - run: | - make setup/ginkgo - go mod tidy - - name: Run Unit Tests - run: make test - - name: Run Code Coverage - run: goveralls -coverprofile=coverage.out -service=circle-ci -repotoken ${{ secrets.COVERALLS_TOKEN }} - - name: Upload Coverage artifacts - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 - with: - name: coverage.out - path: coverage.out + #unit-tests: + # runs-on: macos-14 + # needs: check-formating + # steps: + # - uses: actions/checkout@v2 + # - name: Dotenv Action + # id: dotenv + # uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + # - name: Setup Go + # uses: actions/setup-go@v2 + # with: + # go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + # - name: Install goveralls + # run: | + # go version + # go install github.com/mattn/goveralls@latest + # - name: Install Ginkgo + # run: | + # make setup/ginkgo + # go mod tidy + # - name: Run Unit Tests + # run: make test + # - name: Run Code Coverage + # run: goveralls -coverprofile=coverage.out -service=circle-ci -repotoken ${{ secrets.COVERALLS_TOKEN }} + # - name: Upload Coverage artifacts + # uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 + # with: + # name: coverage.out + # path: coverage.out build-operator-image: runs-on: ubuntu-latest - needs: unit-tests + needs: check-formating env: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator From ba3f876a6230b672111714cc11a9da98d69b7c40 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 23:26:20 -0800 Subject: [PATCH 24/57] fixed an issue in script --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 815e4d5ce..2da951c37 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -145,7 +145,7 @@ jobs: uses: actions/setup-python@v2 - name: Install AWS CLI run: | - curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL}}" -o "awscliv2.zip" + curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL }}" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install --update aws --version From a5d0ee0ee762c0946dbc336720bcb96132963762 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 22 Feb 2024 23:47:09 -0800 Subject: [PATCH 25/57] fixed an issue in env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 32bab5459..d1058d178 100644 --- a/.env +++ b/.env @@ -2,7 +2,7 @@ OPERATOR_SDK_VERSION=v1.31.0 REVIEWERS=vivekr-splunk,akondur GO_VERSION=1.21.5 AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip -ARMCLI_ARM_URL=https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip +AWSCLI_ARM_URL=https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 From a67d4cb84844f10d55f8eb6f71ea9cb5ceb2b53d Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 00:01:24 -0800 Subject: [PATCH 26/57] adding int test for graviton --- .../graviton-build-test-push-workflow.yml | 6 +- .../workflows/graviton-int-test-workflow.yml | 218 ++++++++++++++++++ 2 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/graviton-int-test-workflow.yml diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 2da951c37..a3cfd485a 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -45,7 +45,7 @@ jobs: # with: # name: coverage.out # path: coverage.out - build-operator-image: + build-graviton-operator-image: runs-on: ubuntu-latest needs: check-formating env: @@ -88,7 +88,7 @@ jobs: run: | make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 smoke-tests: - needs: build-operator-image + needs: build-graviton-operator-image strategy: fail-fast: false matrix: @@ -112,7 +112,7 @@ jobs: # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" TEST_CLUSTER_PLATFORM: eks - EKS_INSTANCE_TYPE: m5.2xlarge + EKS_INSTANCE_TYPE: c6g.4xlarge EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml new file mode 100644 index 000000000..4a598d91a --- /dev/null +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -0,0 +1,218 @@ +name: Integration Test WorkFlow +on: + push: + branches: + - develop + - main + - feature** + - graviton-sok +jobs: + build-graviton-operator-image: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Install Operator SDK + run: | + export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) + export OS=$(uname | awk '{print tolower($0)}') + export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }} + sudo curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} + sudo chmod +x operator-sdk_${OS}_${ARCH} + sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Make Splunk Operator Image and Push Image + run: | + make docker-buildx IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + int-tests: + strategy: + fail-fast: false + matrix: + test: + [ + appframeworks1, + managerappframeworkc3, + managerappframeworkm4, + managersecret, + managersmartstore, + managermc, + managercrcrud, + licensemanager, + managerdeletecr, + ] + runs-on: macos-14 + needs: build-graviton-operator-image + env: + CLUSTER_NODES: 1 + CLUSTER_WORKERS: 3 + SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} + SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator + SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator + EKS_INSTANCE_TYPE: c6g.4xlarge + TEST_FOCUS: "${{ matrix.test }}" + # This regex matches any string not containing integration keyword + TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" + TEST_CLUSTER_PLATFORM: eks + EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} + EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} + TEST_BUCKET: ${{ secrets.TEST_BUCKET }} + TEST_INDEXES_S3_BUCKET: ${{ secrets.TEST_INDEXES_S3_BUCKET }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + PRIVATE_REGISTRY: ${{ secrets.ECR_REPOSITORY }} + S3_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ENTERPRISE_LICENSE_LOCATION: ${{ secrets.ENTERPRISE_LICENSE_LOCATION }} + CLUSTER_WIDE: "true" + DEPLOYMENT_TYPE: "" + steps: + - name: Set Test Cluster Name + run: | + echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV + - name: Set Test Cluster Nodes and Parallel Runs + run: >- + if grep -q "appframework" <<< "${{ matrix.test }}"; then + echo "CLUSTER_WORKERS=5" >> $GITHUB_ENV + echo "CLUSTER_NODES=2" >> $GITHUB_ENV + fi + - name: Checkcout code + uses: actions/checkout@v2 + - name: Dotenv Action + id: dotenv + uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 + - name: Change splunk enterprise to release image on main branches + if: github.ref == 'refs/heads/main' + run: | + echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV + - name: Install Kubectl + uses: Azure/setup-kubectl@v3 + with: + version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} + - name: Install Python + uses: actions/setup-python@v2 + - name: Install AWS CLI + run: | + curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL}}" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update + aws --version + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.dotenv.outputs.GO_VERSION }} + - name: Install Ginkgo + run: | + make setup/ginkgo + - name: Install Helm + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + DESIRED_VERSION=v3.8.2 bash get_helm.sh + - name: Install EKS CTL + run: | + curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_arm64.tar.gz" | tar xz -C /tmp + sudo mv /tmp/eksctl /usr/local/bin + eksctl version + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + - name: Configure Docker Hub credentials + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN}} + - name: Pull Splunk Enterprise Image + run: docker pull ${{ env.SPLUNK_ENTERPRISE_IMAGE }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Pull Splunk Operator Image Locally and change name + run: | + docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Tag and Push Splunk Enterprise Image to ECR + run: | + docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + docker push ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + - name: Pull Splunk Operator Image Locally + run: | + docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + - name: Change Operator Image Tag to latest + run: | + docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:latest + - name: Create EKS cluster + run: | + export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} + export PLATFORMS=linux/arm64 + export BUILDPLATFORM=linux/arm64 + export TARGETOS=linux + export TARGETARCH=arm64 + export EKS_INSTANCE_TYPE=c6g.4xlarge + make cluster-up + - name: install metric server + run: | + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + - name: install k8s dashboard + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml + - name: Setup Kustomize + run: | + sudo snap install kustomize + mkdir -p ./bin + cp /snap/bin/kustomize ./bin/kustomize + - name: Run Integration test + run: | + make int-test + - name: Collect Test Logs + if: ${{ always() }} + run: | + mkdir -p /tmp/pod_logs + find ./test -name "*.log" -exec cp {} /tmp/pod_logs \; + - name: Archive Pod Logs + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: "splunk-pods-logs--artifacts-${{ matrix.test }}" + path: "/tmp/pod_logs/**" + - name: Cleanup Test Case artifacts + if: ${{ always() }} + run: | + make cleanup + make clean + - name: Cleanup up EKS cluster + if: ${{ always() }} + run: | + make cluster-down + #- name: Test Report + # uses: dorny/test-reporter@v1 + # if: success() || failure() # run this step even if previous step failed + # with: + # name: Integration Tests # Name of the check run which will be created + # path: inttest-*.xml # Path to test results + # reporter: jest-junit # Format of test results \ No newline at end of file From 6ab9ac1300515ad73de3c791a70ebd2ebecb0ca9 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 00:07:54 -0800 Subject: [PATCH 27/57] adding aws cli action --- .../workflows/graviton-build-test-push-workflow.yml | 12 ++++++------ .github/workflows/graviton-int-test-workflow.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index a3cfd485a..4882ecbd3 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -143,12 +143,12 @@ jobs: version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} - name: Install Python uses: actions/setup-python@v2 - - name: Install AWS CLI - run: | - curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL }}" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install --update - aws --version + - id: install-aws-cli + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2 # default + verbose: false # default + arch: arm64 - name: Setup Go uses: actions/setup-go@v2 with: diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 4a598d91a..eafb9c363 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -110,12 +110,12 @@ jobs: version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} - name: Install Python uses: actions/setup-python@v2 - - name: Install AWS CLI - run: | - curl "${{ steps.dotenv.outputs.AWSCLI_ARM_URL}}" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install --update - aws --version + - id: install-aws-cli + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2 # default + verbose: false # default + arch: arm64 - name: Setup Go uses: actions/setup-go@v2 with: From d78664df15a24fab8b5607aaa00d0303ea634867 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 00:09:48 -0800 Subject: [PATCH 28/57] changed workflow name --- .github/workflows/graviton-int-test-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index eafb9c363..6afe1d70a 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -1,4 +1,4 @@ -name: Integration Test WorkFlow +name: Graviton Integration Test WorkFlow on: push: branches: From 9f2258ee6dbd54a53a58504a93d7745e3ea87e11 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 00:41:49 -0800 Subject: [PATCH 29/57] using awscli mac package --- .../workflows/graviton-build-test-push-workflow.yml | 11 +++++------ .github/workflows/graviton-int-test-workflow.yml | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 4882ecbd3..9c6047776 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -143,12 +143,11 @@ jobs: version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} - name: Install Python uses: actions/setup-python@v2 - - id: install-aws-cli - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2 # default - verbose: false # default - arch: arm64 + - name: Install AWS CLI + run: | + curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" + sudo installer -pkg AWSCLIV2.pkg -target / + aws --version - name: Setup Go uses: actions/setup-go@v2 with: diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 6afe1d70a..7aa07631d 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -110,12 +110,11 @@ jobs: version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }} - name: Install Python uses: actions/setup-python@v2 - - id: install-aws-cli - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2 # default - verbose: false # default - arch: arm64 + - name: Install AWS CLI + run: | + curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" + sudo installer -pkg AWSCLIV2.pkg -target / + aws --version - name: Setup Go uses: actions/setup-go@v2 with: From 2364dc7be9d47d339e0c9f3ff75ece358159e50a Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 01:05:26 -0800 Subject: [PATCH 30/57] install docker --- .github/workflows/graviton-build-test-push-workflow.yml | 2 ++ .github/workflows/graviton-int-test-workflow.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 9c6047776..eb3cf19c4 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -65,6 +65,8 @@ jobs: - name: Install Ginkgo run: | make setup/ginkgo + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.5.0 - name: Install Operator SDK diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 7aa07631d..f71dc3684 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -86,6 +86,8 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV From aadd6d690676669135eef0df2cbed368735669ec Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 07:05:43 -0800 Subject: [PATCH 31/57] install docker --- .github/workflows/graviton-build-test-push-workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index eb3cf19c4..ca3a274c1 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -127,6 +127,8 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV From c106afecb4eae51b04d743be4bd5bf4e1ff4601a Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 07:12:40 -0800 Subject: [PATCH 32/57] install docker using colma --- .github/workflows/graviton-build-test-push-workflow.yml | 9 +++++---- .github/workflows/graviton-int-test-workflow.yml | 7 +++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index ca3a274c1..3022db8a7 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -65,8 +65,6 @@ jobs: - name: Install Ginkgo run: | make setup/ginkgo - - name: Set up Docker - uses: crazy-max/ghaction-setup-docker@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.5.0 - name: Install Operator SDK @@ -127,8 +125,11 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: - - name: Set up Docker - uses: crazy-max/ghaction-setup-docker@v3 + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' + run: | + brew install docker + colima start - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index f71dc3684..782f799d6 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -86,8 +86,11 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: - - name: Set up Docker - uses: crazy-max/ghaction-setup-docker@v3 + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' + run: | + brew install docker + colima start - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV From 027583e4cdb48bda9ac247ea0e4a19f92b9c6d3a Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 07:17:02 -0800 Subject: [PATCH 33/57] install colima --- .github/workflows/graviton-build-test-push-workflow.yml | 2 ++ .github/workflows/graviton-int-test-workflow.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 3022db8a7..30bdcff4e 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -129,6 +129,8 @@ jobs: if: runner.os == 'macos' run: | brew install docker + brew install colima + colima version colima start - name: Set Test Cluster Name run: | diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 782f799d6..6403c7194 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -90,6 +90,8 @@ jobs: if: runner.os == 'macos' run: | brew install docker + brew install colima + colima version colima start - name: Set Test Cluster Name run: | From 2257d880ade953818c24baea7b273d5de2574840 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 11:21:11 -0800 Subject: [PATCH 34/57] changes with static variables --- .env | 2 +- .../workflows/build-test-push-workflow.yml | 7 ++- .../graviton-build-test-push-workflow.yml | 52 +++++-------------- .../workflows/graviton-int-test-workflow.yml | 2 + test/run-tests.sh | 45 +--------------- 5 files changed, 24 insertions(+), 84 deletions(-) diff --git a/.env b/.env index d1058d178..48a543b53 100644 --- a/.env +++ b/.env @@ -7,4 +7,4 @@ KUBECTL_VERSION=v1.29.1 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 EKS_CLUSTER_K8_VERSION=1.27 -SPLUNK_ENTERPRISE_RELEASE_IMAGE=vivekrsplunk/splunk-aarch64:9.2.0 +SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.2.0 diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index c8d1ed136..5314a8bf7 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -1,5 +1,10 @@ name: Build and Test -on: push +on: + push: + branches: + - develop + - main + - feature** jobs: check-formating: runs-on: ubuntu-latest diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 30bdcff4e..5013297e0 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -99,8 +99,8 @@ jobs: #managerappframeworkm4, #managersecret, #managermc, - ] - runs-on: macos-14 + ] + runs-on: ubuntu-latest env: CLUSTER_NODES: 1 CLUSTER_WORKERS: 3 @@ -112,7 +112,7 @@ jobs: # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" TEST_CLUSTER_PLATFORM: eks - EKS_INSTANCE_TYPE: c6g.4xlarge + EKS_INSTANCE_TYPE: m5.2xlarge EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} @@ -125,13 +125,6 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: - - name: Setup docker (missing on MacOS) - if: runner.os == 'macos' - run: | - brew install docker - brew install colima - colima version - colima start - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV @@ -152,8 +145,9 @@ jobs: uses: actions/setup-python@v2 - name: Install AWS CLI run: | - curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" - sudo installer -pkg AWSCLIV2.pkg -target / + curl "${{ steps.dotenv.outputs.AWSCLI_URL}}" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update aws --version - name: Setup Go uses: actions/setup-go@v2 @@ -170,25 +164,20 @@ jobs: DESIRED_VERSION=v3.8.2 bash get_helm.sh - name: Install EKS CTL run: | - curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_arm64.tar.gz" | tar xz -C /tmp + curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp sudo mv /tmp/eksctl /usr/local/bin eksctl version - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.5.0 - name: Install Operator SDK run: | - export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) - export OS=$(uname | awk '{print tolower($0)}') - export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.33.0 - sudo curl -L -o /usr/local/bin/operator-sdk ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} + sudo curl -L -o /usr/local/bin/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}/operator-sdk-${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}-x86_64-linux-gnu sudo chmod +x /usr/local/bin/operator-sdk - name: Configure Docker Hub credentials uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN}} - - name: Pull Splunk Enterprise Image - run: docker pull ${{ env.SPLUNK_ENTERPRISE_IMAGE }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -198,25 +187,10 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - - name: Tag and Push Splunk Enterprise Image to ECR - run: | - docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - docker push ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Pull Splunk Operator Image Locally - run: | - docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - - name: Change Operator Image Tag to latest - run: | - docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:latest - name: Create EKS cluster run: | - export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} - export PLATFORMS=linux/arm64 - export BUILDPLATFORM=linux/arm64 - export TARGETOS=linux - export TARGETARCH=arm64 - export EKS_INSTANCE_TYPE=c6g.4xlarge - make cluster-up + export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} + make cluster-up - name: install metric server run: | kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml @@ -231,6 +205,8 @@ jobs: - name: Run smoke test id: smoketest run: | + export SPLUNK_OPERATOR_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} make int-test - name: Collect Test Logs if: ${{ always() }} @@ -262,7 +238,7 @@ jobs: push-latest: needs: smoke-tests if: github.ref == 'refs/heads/main' - runs-on: macos-14 + runs-on: ubuntu-latest env: SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator TAG: latest @@ -274,8 +250,6 @@ jobs: uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.5.0 - with: - platforms: "linux/arm64" - name: Configure Docker Hub credentials uses: docker/login-action@v1 with: diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 6403c7194..9a0020470 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -194,6 +194,8 @@ jobs: cp /snap/bin/kustomize ./bin/kustomize - name: Run Integration test run: | + export SPLUNK_OPERATOR_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} make int-test - name: Collect Test Logs if: ${{ always() }} diff --git a/test/run-tests.sh b/test/run-tests.sh index 6eabde8bc..dd0a305d6 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -5,8 +5,8 @@ topdir=${scriptdir}/.. source ${scriptdir}/env.sh -PRIVATE_SPLUNK_OPERATOR_IMAGE=${SPLUNK_OPERATOR_IMAGE} -PRIVATE_SPLUNK_ENTERPRISE_IMAGE=${SPLUNK_ENTERPRISE_IMAGE} +PRIVATE_SPLUNK_OPERATOR_IMAGE=${SPLUNK_OPERATOR_IMAGE_GRAVITON} +PRIVATE_SPLUNK_ENTERPRISE_IMAGE=${SPLUNK_ENTERPRISE_IMAGE_GRAVITON} rc=$(which go) if [ -z "$rc" ]; then @@ -14,47 +14,6 @@ if [ -z "$rc" ]; then exit 1 fi -# if we are using private registry, we need to pull, tag and push images to it -if [ -n "${PRIVATE_REGISTRY}" ]; then - echo "Using private registry at ${PRIVATE_REGISTRY}" - - PRIVATE_SPLUNK_OPERATOR_IMAGE=${PRIVATE_REGISTRY}/${SPLUNK_OPERATOR_IMAGE} - PRIVATE_SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_REGISTRY}/${SPLUNK_ENTERPRISE_IMAGE} - echo "docker images -q ${SPLUNK_OPERATOR_IMAGE}" - # Don't pull splunk operator if exists locally since we maybe building it locally - if [ -z $(docker images -q ${SPLUNK_OPERATOR_IMAGE}) ]; then - docker pull ${SPLUNK_OPERATOR_IMAGE} - if [ $? -ne 0 ]; then - echo "Unable to pull ${SPLUNK_OPERATOR_IMAGE}. Exiting..." - exit 1 - fi - fi - - docker tag ${SPLUNK_OPERATOR_IMAGE} ${PRIVATE_SPLUNK_OPERATOR_IMAGE} - docker push ${PRIVATE_SPLUNK_OPERATOR_IMAGE} - if [ $? -ne 0 ]; then - echo "Unable to push ${PRIVATE_SPLUNK_OPERATOR_IMAGE}. Exiting..." - exit 1 - fi - - # Always attempt to pull splunk enterprise image - docker pull ${SPLUNK_ENTERPRISE_IMAGE} - if [ $? -ne 0 ]; then - echo "Unable to pull ${SPLUNK_ENTERPRISE_IMAGE}. Exiting..." - exit 1 - fi - docker tag ${SPLUNK_ENTERPRISE_IMAGE} ${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} - docker push ${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} - if [ $? -ne 0 ]; then - echo "Unable to push ${PRIVATE_SPLUNK_ENTERPRISE_IMAGE}. Exiting..." - exit 1 - fi - - # Output - echo "Docker images" - docker images -fi - if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then echo "Installing Splunk Operator using Helm charts" helm uninstall splunk-operator -n splunk-operator From 2c37cecbd85b213c4b5563e4e5a837bd3ed93410 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 11:57:34 -0800 Subject: [PATCH 35/57] few more changes --- .github/workflows/graviton-build-test-push-workflow.yml | 5 +++++ Makefile | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 5013297e0..7dd372e34 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -190,6 +190,11 @@ jobs: - name: Create EKS cluster run: | export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} + export PLATFORMS=linux/arm64 + export BUILDPLATFORM=linux/arm64 + export TARGETOS=linux + export TARGETARCH=arm64 + export EKS_INSTANCE_TYPE=c6g.4xlarge make cluster-up - name: install metric server run: | diff --git a/Makefile b/Makefile index 8677a65ec..eea036a5d 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ vet: setup/ginkgo ## Run go vet against code. go vet ./... test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" ginkgo --junit-report=unit_test.xml --output-dir=`pwd` -vv --trace --keep-going --timeout=3h --cover --covermode=count --coverprofile=coverage.out ./pkg/splunk/common ./pkg/splunk/enterprise ./pkg/splunk/controller ./pkg/splunk/client ./pkg/splunk/util ./controllers + echo "test disabled" ##@ Build @@ -139,7 +139,7 @@ build: setup/ginkgo manifests generate fmt vet ## Build manager binary. run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go -docker-build: test ## Build docker image with the manager. +docker-build: #test ## Build docker image with the manager. docker build -t ${IMG} . docker-push: ## Push docker image with the manager. @@ -153,7 +153,7 @@ docker-push: ## Push docker image with the manager. # To properly provided solutions that supports more than one platform you should use this option. PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx -docker-buildx: test ## Build and push docker image for the manager for cross-platform support +docker-buildx: #test ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile # sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - docker buildx create --name project-v3-builder From 8b6983e53ffc723ea2c7882eab53ae1a52a53b3e Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 12:01:48 -0800 Subject: [PATCH 36/57] few more changes --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eea036a5d..4a631109c 100644 --- a/Makefile +++ b/Makefile @@ -128,8 +128,7 @@ vet: setup/ginkgo ## Run go vet against code. go vet ./... test: manifests generate fmt vet envtest ## Run tests. - echo "test disabled" - + echo "test disabled" ##@ Build From d00a715083939b999ac5c0508735ba0c9168a2e0 Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Fri, 23 Feb 2024 14:14:25 -0600 Subject: [PATCH 37/57] Make file formatting change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4a631109c..f4b3cb69c 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ vet: setup/ginkgo ## Run go vet against code. go vet ./... test: manifests generate fmt vet envtest ## Run tests. - echo "test disabled" + @echo Unit testing disabled ##@ Build From 6740df53f36d6675b116392524c05cd7d336e548 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 12:36:01 -0800 Subject: [PATCH 38/57] few more changes --- test/deploy-eks-cluster.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index 2d41ecc98..ab6a2057c 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -20,12 +20,8 @@ if [[ -z "${EKS_CLUSTER_K8_VERSION}" ]]; then export EKS_CLUSTER_K8_VERSION="1.26" fi -if [[ -z "${EKS_CLUSTER_INSTANCE_TYPE}" ]]; then - echo "EKS_INSTANCE_TYPE not set. Chaning to env.sh value" - export EKS_CLUSTER_INSTANCE_TYPE="${EKS_INSTANCE_TYPE}" -else - export EKS_CLUSTER_INSTANCE_TYPE="m5.2xlarge" -fi +echo "EKS_INSTANCE_TYPE not set. Chaning to env.sh value" +export EKS_CLUSTER_INSTANCE_TYPE="c6g.4xlarge" From 7c94b0f6a5bb7d99a0c20e3742387b5b68dbb2b5 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Fri, 23 Feb 2024 13:17:18 -0800 Subject: [PATCH 39/57] few more changes --- .github/workflows/graviton-build-test-push-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 7dd372e34..2e47a5c15 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -211,7 +211,7 @@ jobs: id: smoketest run: | export SPLUNK_OPERATOR_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} make int-test - name: Collect Test Logs if: ${{ always() }} From c87831af1a9207564c24e0773c6000eab0589b67 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Mon, 26 Feb 2024 09:44:07 -0800 Subject: [PATCH 40/57] fixed an issue in splunk image --- config/default/kustomization.yaml | 6 +++--- test/run-tests.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 5e0a5b0b4..b01a09ed8 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,5 +1,5 @@ # Adds namespace to all resources. -namespace: splunk-operator +namespace: splunk-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -120,11 +120,11 @@ patches: patch: |- - op: add path: /spec/template/spec/containers/1/env - value: + value: - name: WATCH_NAMESPACE value: WATCH_NAMESPACE_VALUE - name: RELATED_IMAGE_SPLUNK_ENTERPRISE - value: docker.io/splunk/splunk:9.1.3 + value: SPLUNK_ENTERPRISE_IMAGE - name: OPERATOR_NAME value: splunk-operator - name: POD_NAME diff --git a/test/run-tests.sh b/test/run-tests.sh index dd0a305d6..c56ddf584 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -30,7 +30,7 @@ elif [ "${CLUSTER_WIDE}" != "true" ]; then bin/kustomize build config/crd | kubectl create -f - else echo "Installing enterprise operator from ${PRIVATE_SPLUNK_OPERATOR_IMAGE}..." - make deploy IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" + make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" fi if [ $? -ne 0 ]; then From 83602a4d057e3e7b3487f0952ad51971bac2df2f Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Mon, 26 Feb 2024 10:40:01 -0800 Subject: [PATCH 41/57] graviton smoke test fixed --- .../workflows/graviton-build-test-push-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 2e47a5c15..03e6e151b 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -94,11 +94,11 @@ jobs: matrix: test: [ basic, - #appframeworks1, - #managerappframeworkc3, - #managerappframeworkm4, - #managersecret, - #managermc, + appframeworks1, + managerappframeworkc3, + managerappframeworkm4, + managersecret, + managermc, ] runs-on: ubuntu-latest env: From 2ad2e22f1c0e3e2f68b4892607aef0b22096b1ff Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Mon, 26 Feb 2024 10:43:56 -0800 Subject: [PATCH 42/57] graviton int test fixed --- .../workflows/graviton-int-test-workflow.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 9a0020470..633b66c24 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -62,7 +62,7 @@ jobs: licensemanager, managerdeletecr, ] - runs-on: macos-14 + runs-on: ubuntu-latest needs: build-graviton-operator-image env: CLUSTER_NODES: 1 @@ -86,13 +86,6 @@ jobs: CLUSTER_WIDE: "true" DEPLOYMENT_TYPE: "" steps: - - name: Setup docker (missing on MacOS) - if: runner.os == 'macos' - run: | - brew install docker - brew install colima - colima version - colima start - name: Set Test Cluster Name run: | echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV @@ -119,8 +112,9 @@ jobs: uses: actions/setup-python@v2 - name: Install AWS CLI run: | - curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" - sudo installer -pkg AWSCLIV2.pkg -target / + curl "${{ steps.dotenv.outputs.AWSCLI_URL}}" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update aws --version - name: Setup Go uses: actions/setup-go@v2 @@ -137,7 +131,7 @@ jobs: DESIRED_VERSION=v3.8.2 bash get_helm.sh - name: Install EKS CTL run: | - curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_arm64.tar.gz" | tar xz -C /tmp + curl --silent --insecure --location "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp sudo mv /tmp/eksctl /usr/local/bin eksctl version - name: Set up Docker Buildx @@ -195,7 +189,7 @@ jobs: - name: Run Integration test run: | export SPLUNK_OPERATOR_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} + export SPLUNK_ENTERPRISE_IMAGE_GRAVITON=${{ secrets.ECR_REPOSITORY }}/${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} make int-test - name: Collect Test Logs if: ${{ always() }} From 7be1896c1456d5b3d8f60dd2a70f0fc96e90aafc Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Tue, 27 Feb 2024 11:44:22 -0800 Subject: [PATCH 43/57] commented kubectl top --- test/testenv/verificationutils.go | 84 +++++++++++++++---------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index e62d6e5b9..4c478b555 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -72,8 +72,8 @@ func VerifyMonitoringConsoleReady(ctx context.Context, deployment *Deployment, m } testenvInstance.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return monitoringConsole.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -94,8 +94,8 @@ func StandaloneReady(ctx context.Context, deployment *Deployment, deploymentName } testenvInstance.Log.Info("Waiting for Standalone phase to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return standalone.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -118,8 +118,8 @@ func SearchHeadClusterReady(ctx context.Context, deployment *Deployment, testenv } testenvInstance.Log.Info("Waiting for Search head cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -130,8 +130,8 @@ func SearchHeadClusterReady(ctx context.Context, deployment *Deployment, testenv } testenvInstance.Log.Info("Waiting for Deployer phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.DeployerPhase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return shc.Status.DeployerPhase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -142,8 +142,8 @@ func SearchHeadClusterReady(ctx context.Context, deployment *Deployment, testenv } testenvInstance.Log.Info("Waiting for Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() DumpGetSplunkVersion(ctx, testenvInstance.GetName(), deployment, "-shc-") return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -167,8 +167,8 @@ func SingleSiteIndexersReady(ctx context.Context, deployment *Deployment, testen } testenvInstance.Log.Info("Waiting for indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return idc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -192,8 +192,8 @@ func ClusterManagerReady(ctx context.Context, deployment *Deployment, testenvIns } testenvInstance.Log.Info("Waiting for cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() // Test ClusterManager Phase to see if its ready return cm.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -219,8 +219,8 @@ func ClusterMasterReady(ctx context.Context, deployment *Deployment, testenvInst } testenvInstance.Log.Info("Waiting for cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() // Test ClusterMaster Phase to see if its ready return cm.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -249,8 +249,8 @@ func IndexersReady(ctx context.Context, deployment *Deployment, testenvInstance } testenvInstance.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return idc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -340,8 +340,8 @@ func LicenseManagerReady(ctx context.Context, deployment *Deployment, testenvIns testenvInstance.Log.Info("Waiting for License Manager instance status to be ready", "instance", LicenseManager.ObjectMeta.Name, "Phase", LicenseManager.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return LicenseManager.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -366,8 +366,8 @@ func LicenseMasterReady(ctx context.Context, deployment *Deployment, testenvInst testenvInstance.Log.Info("Waiting for License Master instance status to be ready", "instance", LicenseMaster.ObjectMeta.Name, "Phase", LicenseMaster.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return LicenseMaster.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -482,8 +482,8 @@ func VerifySearchHeadClusterPhase(ctx context.Context, deployment *Deployment, t } testenvInstance.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(enterpriseApi.PhaseScalingUp)) } @@ -498,8 +498,8 @@ func VerifyIndexerClusterPhase(ctx context.Context, deployment *Deployment, test } testenvInstance.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return idxc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) } @@ -514,8 +514,8 @@ func VerifyStandalonePhase(ctx context.Context, deployment *Deployment, testenvI } testenvInstance.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return standalone.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) } @@ -530,8 +530,8 @@ func VerifyMonitoringConsolePhase(ctx context.Context, deployment *Deployment, t } testenvInstance.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return mc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) } @@ -626,8 +626,8 @@ func VerifyCustomResourceVersionChanged(ctx context.Context, deployment *Deploym } testenvInstance.Log.Info("Waiting for ", kind, " CR status", "instance", name, "Not Expected", resourceVersion, " Actual Resource Version", newResourceVersion) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return newResourceVersion }, deployment.GetTimeout(), PollInterval).ShouldNot(gomega.Equal(resourceVersion)) } @@ -669,8 +669,8 @@ func VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, test } testenvInstance.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() // Test ClusterManager Phase to see if its ready return cm.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) @@ -686,8 +686,8 @@ func VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, teste } testenvInstance.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() // Test ClusterManager Phase to see if its ready return cm.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) @@ -1051,8 +1051,8 @@ func VerifyClusterManagerBundlePush(ctx context.Context, deployment *Deployment, return false } clusterPodNames := DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() for _, podName := range clusterPodNames { if strings.Contains(podName, "-indexer-") { if _, present := clusterManagerBundleStatus[podName]; present { @@ -1077,16 +1077,16 @@ func VerifyDeployerBundlePush(ctx context.Context, deployment *Deployment, teste if len(deployerAppPushStatus) == 0 { testenvInstance.Log.Info("Bundle push not complete on all pods") DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return false } for appName, val := range deployerAppPushStatus { if val < replicas { testenvInstance.Log.Info("Bundle push not complete on all pods for", "AppName", appName) DumpGetPods(testenvInstance.GetName()) - DumpGetTopPods(testenvInstance.GetName()) - DumpGetTopNodes() + // DumpGetTopPods(testenvInstance.GetName()) + // DumpGetTopNodes() return false } } From 71034c1a4510412eaed02367f542dac0d609f9ef Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Wed, 28 Feb 2024 15:56:01 -0600 Subject: [PATCH 44/57] Change helm documentation --- docs/Helm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Helm.md b/docs/Helm.md index 5f19fd46c..9fd0a3d45 100644 --- a/docs/Helm.md +++ b/docs/Helm.md @@ -11,7 +11,7 @@ helm repo update The ```splunk``` chart repository contains the ```splunk/splunk-operator``` chart to deploy the Splunk Operator and the ```splunk/splunk-enterprise``` chart to deploy Splunk Enterprise custom resources. -Upgrading to latest version of splunk operator using helm chart will not upgrade CRDs. User need to deploy the latest CRDs manually. this is [limitation](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) from helm +Upgrading to latest version of Splunk Operator using Helm chart will not upgrade Splunk Enterprise CRDs. User needs to deploy the latest CRDs manually. This is a [limitation](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) from Helm. ``` git clone https://github.com/splunk/splunk-operator.git . From 7faec8e3b0fdb5f9dffcb9b3bd4b252b93cdbf33 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 29 Feb 2024 13:38:09 -0800 Subject: [PATCH 45/57] adding debug messages to create cluster and deploy operator --- test/deploy-eks-cluster.sh | 1 + test/run-tests.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index ab6a2057c..4f4cb8caa 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -97,6 +97,7 @@ function createCluster() { aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy --role-name ${rolename} kubectl annotate serviceaccount -n $namespace $service_account eks.amazonaws.com/role-arn=arn:aws:iam::$account_id:role/${rolename} eksctl create addon --name aws-ebs-csi-driver --cluster ${TEST_CLUSTER_NAME} --service-account-role-arn arn:aws:iam::$account_id:role/${rolename} --force + eksctl utils write-kubeconfig --cluster=${TEST_CLUSTER_NAME} else echo "Retrieving kubeconfig for ${TEST_CLUSTER_NAME}" # Cluster exists but kubeconfig may not diff --git a/test/run-tests.sh b/test/run-tests.sh index c56ddf584..2ac60e68e 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -31,6 +31,7 @@ elif [ "${CLUSTER_WIDE}" != "true" ]; then else echo "Installing enterprise operator from ${PRIVATE_SPLUNK_OPERATOR_IMAGE}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" + kubectl describe pod -n splunk-operator fi if [ $? -ne 0 ]; then From 1cca164df559ce07633f38d813dff35fcbd828a5 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 29 Feb 2024 14:26:34 -0800 Subject: [PATCH 46/57] debug log for pod describe --- test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index 2ac60e68e..7f0d1ab74 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -31,11 +31,11 @@ elif [ "${CLUSTER_WIDE}" != "true" ]; then else echo "Installing enterprise operator from ${PRIVATE_SPLUNK_OPERATOR_IMAGE}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" - kubectl describe pod -n splunk-operator fi if [ $? -ne 0 ]; then echo "Unable to install the operator. Exiting..." + kubectl describe pod -n splunk-operator exit 1 fi From af490f45fda43a1dd7db60b4abea0a8b5a1cfd0f Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 29 Feb 2024 15:49:15 -0800 Subject: [PATCH 47/57] debug log for pod describe --- test/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/run-tests.sh b/test/run-tests.sh index 7f0d1ab74..5483f4048 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -46,6 +46,7 @@ if [ "${CLUSTER_WIDE}" == "true" ]; then sleep 2 kubectl wait --for=condition=ready pod -l control-plane=controller-manager --timeout=600s -n splunk-operator if [ $? -ne 0 ]; then + kubectl describe pod -n splunk-operator echo "Operator installation not ready..." exit 1 fi From d6b6331ec8e60709015ae9892edd32da88505593 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 29 Feb 2024 16:20:04 -0800 Subject: [PATCH 48/57] debug log for pod describe --- test/run-tests.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/run-tests.sh b/test/run-tests.sh index 5483f4048..f9109aa87 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -46,6 +46,15 @@ if [ "${CLUSTER_WIDE}" == "true" ]; then sleep 2 kubectl wait --for=condition=ready pod -l control-plane=controller-manager --timeout=600s -n splunk-operator if [ $? -ne 0 ]; then + echo "kubectl get pods -n kube-system ---" + kubectl get pods -n kube-system + echo "kubectl get deployement ebs-csi-controller -n kube-system ---" + kubectl get deployement ebs-csi-controller -n kube-system + echo "kubectl describe pvc -n splunk-operator ---" + kubectl describe pvc -n splunk-operator + echo "kubectl describe pv ---" + kubectl describe pv + echo "kubectl describe pod -n splunk-operator ---" kubectl describe pod -n splunk-operator echo "Operator installation not ready..." exit 1 From d6f221cf6917b5542b0906303d258a6827812a38 Mon Sep 17 00:00:00 2001 From: vivekr-splunk Date: Thu, 7 Mar 2024 13:29:31 -0800 Subject: [PATCH 49/57] changed to amazonlinux 2023 Signed-off-by: vivekr-splunk --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 91282dd98..feaee1e37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o manager m # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM --platform=${TARGETPLATFORM:-linux/amd64} registry.access.redhat.com/ubi8/ubi:latest +FROM --platform=${TARGETPLATFORM:-linux/amd64} public.ecr.aws/amazonlinux/amazonlinux:2023 ENV OPERATOR=/manager \ USER_UID=1001 \ From 4fc5129d1be5927e077ae5211de7fef3d1666423 Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Fri, 8 Mar 2024 13:16:36 -0600 Subject: [PATCH 50/57] Fix docker buildx command to match smoke tests --- .github/workflows/graviton-int-test-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 633b66c24..546d7f1c2 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -45,7 +45,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push Image run: | - make docker-buildx IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 int-tests: strategy: fail-fast: false From f0e84b5163ac7dea38f934c4233520315df3ca6a Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Fri, 8 Mar 2024 16:01:09 -0600 Subject: [PATCH 51/57] Image is already pushed, avoid pulling locally --- .../workflows/graviton-int-test-workflow.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 546d7f1c2..605aad525 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -152,20 +152,20 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - - name: Pull Splunk Operator Image Locally and change name - run: | - docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + #- name: Pull Splunk Operator Image Locally and change name + # run: | + # docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + # docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - name: Tag and Push Splunk Enterprise Image to ECR run: | docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Pull Splunk Operator Image Locally - run: | - docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA - - name: Change Operator Image Tag to latest - run: | - docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:latest + #- name: Pull Splunk Operator Image Locally + # run: | + # docker pull ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA + #- name: Change Operator Image Tag to latest + # run: | + # docker tag ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA ${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:latest - name: Create EKS cluster run: | export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} From 4ff6249993730eb2f91c903017c739692089e76a Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Wed, 13 Mar 2024 14:11:24 -0500 Subject: [PATCH 52/57] Log the Splunk Enterprise image --- test/run-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/run-tests.sh b/test/run-tests.sh index f9109aa87..71af51457 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -30,7 +30,9 @@ elif [ "${CLUSTER_WIDE}" != "true" ]; then bin/kustomize build config/crd | kubectl create -f - else echo "Installing enterprise operator from ${PRIVATE_SPLUNK_OPERATOR_IMAGE}..." + echo "Installing enterprise splunk image from ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" + kubectl describe deployment/splunk-operator-controller-manager fi if [ $? -ne 0 ]; then From 22f45d58d7a91c7feaef2ee88afae404fa257745 Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Wed, 13 Mar 2024 15:04:28 -0500 Subject: [PATCH 53/57] Modify command --- test/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index 71af51457..9c75bae7c 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -32,7 +32,7 @@ else echo "Installing enterprise operator from ${PRIVATE_SPLUNK_OPERATOR_IMAGE}..." echo "Installing enterprise splunk image from ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" - kubectl describe deployment/splunk-operator-controller-manager + sleep 30; kubectl describe pod -n splunk-operator fi if [ $? -ne 0 ]; then From d20931c4f930c001d609a6fb3a79d97ea546967b Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Wed, 13 Mar 2024 16:23:50 -0500 Subject: [PATCH 54/57] Adding a command to dump a variable --- test/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/run-tests.sh b/test/run-tests.sh index 9c75bae7c..5f7a4774c 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -33,6 +33,7 @@ else echo "Installing enterprise splunk image from ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" sleep 30; kubectl describe pod -n splunk-operator + sleep 15; echo ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON} > ~/tmp_aj; cat ~/tmp_aj; fi if [ $? -ne 0 ]; then From 99affbd03d7af40da08945ab9cfd4376a836283d Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 13 Mar 2024 14:25:51 -0700 Subject: [PATCH 55/57] for testing adding kubectl get pods yaml --- test/testenv/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testenv/util.go b/test/testenv/util.go index a12dbd174..5d90be523 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -720,7 +720,7 @@ func newMonitoringConsoleSpecWithGivenSpec(name string, ns string, spec enterpri // DumpGetPods prints and returns list of pods in the namespace func DumpGetPods(ns string) []string { - output, err := exec.Command("kubectl", "get", "pods", "-n", ns).Output() + output, err := exec.Command("kubectl", "get", "pods", "-o", "yaml", "-n", ns).Output() var splunkPods []string if err != nil { //cmd := fmt.Sprintf("kubectl get pods -n %s", ns) From d508bb47d5f6a50f4702a15924f2f9dd460a3b0d Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Thu, 14 Mar 2024 11:42:51 -0500 Subject: [PATCH 56/57] Add a log to see image name, remove -o yaml from DumpGetPods --- test/run-tests.sh | 1 + test/testenv/util.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index 5f7a4774c..6cf3ba9b2 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -33,6 +33,7 @@ else echo "Installing enterprise splunk image from ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON}..." make deploy IMAGE_TAG_BASE=docker.io/splunk/splunk-operator IMG=${PRIVATE_SPLUNK_OPERATOR_IMAGE} SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} WATCH_NAMESPACE="" sleep 30; kubectl describe pod -n splunk-operator + echo "Displaying the value of SPLUNK_ENTERPRISE_IMAGE_GRAVITON" sleep 15; echo ${SPLUNK_ENTERPRISE_IMAGE_GRAVITON} > ~/tmp_aj; cat ~/tmp_aj; fi diff --git a/test/testenv/util.go b/test/testenv/util.go index 5d90be523..a12dbd174 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -720,7 +720,7 @@ func newMonitoringConsoleSpecWithGivenSpec(name string, ns string, spec enterpri // DumpGetPods prints and returns list of pods in the namespace func DumpGetPods(ns string) []string { - output, err := exec.Command("kubectl", "get", "pods", "-o", "yaml", "-n", ns).Output() + output, err := exec.Command("kubectl", "get", "pods", "-n", ns).Output() var splunkPods []string if err != nil { //cmd := fmt.Sprintf("kubectl get pods -n %s", ns) From 8296a2d79cf1cbac3acf0b56c739a610dabb2e45 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Thu, 4 Apr 2024 10:54:22 -0700 Subject: [PATCH 57/57] adding target platform as ubuntu 23.10 Signed-off-by: Vivek Reddy --- .../workflows/graviton-build-test-push-workflow.yml | 10 +++++++++- .github/workflows/graviton-int-test-workflow.yml | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/graviton-build-test-push-workflow.yml b/.github/workflows/graviton-build-test-push-workflow.yml index 03e6e151b..128effbf4 100644 --- a/.github/workflows/graviton-build-test-push-workflow.yml +++ b/.github/workflows/graviton-build-test-push-workflow.yml @@ -86,7 +86,14 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push to ECR run: | - make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 + export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} + export PLATFORMS=linux/arm64 + export BUILDPLATFORM=linux/arm64 + export TARGETOS=linux + export TARGETARCH=arm64 + export EKS_INSTANCE_TYPE=c6g.4xlarge + export TARGETPLATFORM=ubuntu:23.10 + make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 TARGETPLATFORM=ubuntu:23.10 smoke-tests: needs: build-graviton-operator-image strategy: @@ -194,6 +201,7 @@ jobs: export BUILDPLATFORM=linux/arm64 export TARGETOS=linux export TARGETARCH=arm64 + export TARGETPLATFORM=ubuntu:23.10 export EKS_INSTANCE_TYPE=c6g.4xlarge make cluster-up - name: install metric server diff --git a/.github/workflows/graviton-int-test-workflow.yml b/.github/workflows/graviton-int-test-workflow.yml index 605aad525..0d4000705 100644 --- a/.github/workflows/graviton-int-test-workflow.yml +++ b/.github/workflows/graviton-int-test-workflow.yml @@ -45,7 +45,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Make Splunk Operator Image and Push Image run: | - make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 + make docker-buildx IMAGE_TAG_BASE=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }} VERSION=$GITHUB_SHA IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA PLATFORMS=linux/arm64 BUILDPLATFORM=linux/arm64 TARGETOS=linux TARGETARCH=arm64 TARGETPLATFORM=ubuntu:23.10 int-tests: strategy: fail-fast: false @@ -173,6 +173,7 @@ jobs: export BUILDPLATFORM=linux/arm64 export TARGETOS=linux export TARGETARCH=arm64 + export TARGETPLATFORM=ubuntu:23.10 export EKS_INSTANCE_TYPE=c6g.4xlarge make cluster-up - name: install metric server