From ab8f36089a635e820a7d4549beefea06ac5aba16 Mon Sep 17 00:00:00 2001 From: ntishchauhan0022 Date: Tue, 18 Jul 2023 17:56:26 +0530 Subject: [PATCH] making reviewed changes Signed-off-by: ntishchauhan0022 --- .circleci/config.yml | 22 ++- .../airflow-operator-release-to-pypi.yml | 2 +- .github/workflows/airflow-operator.yml | 6 +- .github/workflows/build-release-images.yml | 10 +- .github/workflows/codeql-analysis.yml | 4 +- .github/workflows/go-setup/action.yml | 2 +- .github/workflows/go.yml | 0 .github/workflows/python-client.yml | 2 +- .github/workflows/python-tests/action.yml | 2 +- .goreleaser.yml | 4 +- magefiles/airflow.go | 6 +- magefiles/main.go | 16 ++- magefiles/tests.go | 133 ++++++++++++++++-- 13 files changed, 161 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/go.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index bf3f2192e4a..3ede1fc87f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,17 +160,17 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl mv kubectl /home/circleci/bin/ - make download # no-op if we restored from cache + mage download # no-op if we restored from cache - run: name: Build command: | - make build-ci - make build-docker-full-bundle + mage buildCI + mage buildDockers full-bundle - run: name: e2e test environment setup - command: make tests-e2e-setup + command: mage teste2eSetup - run: name: Enable integration tests @@ -178,15 +178,11 @@ jobs: - run: name: e2e tests - command: make tests-e2e-no-setup - - - run: - name: e2e test environment teardown - command: make tests-e2e-teardown + command: mage testse2eNoSetup - run: name: Prepare reports - command: make junit-report + command: mage junitReport when: always - store_test_results: @@ -288,7 +284,7 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ - make download # no-op if we restored from cache + mage download # no-op if we restored from cache - run: name: Build armadactl release artifacts @@ -429,7 +425,7 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ - make download # no-op if we restored from cache + mage download # no-op if we restored from cache - run: name: Push dotnet clients to nuget @@ -440,7 +436,7 @@ jobs: go mod download go run github.com/magefile/mage@v1.14.0 -v BootstrapTools go run github.com/magefile/mage@v1.14.0 -v proto - RELEASE_TAG=${CIRCLE_TAG#"v"} make push-nuget + RELEASE_TAG=${CIRCLE_TAG#"v"} mage pushNuget - store_artifacts: path: bin/client/DotNet diff --git a/.github/workflows/airflow-operator-release-to-pypi.yml b/.github/workflows/airflow-operator-release-to-pypi.yml index 3d7ac357d75..3fcd6fe8cfd 100644 --- a/.github/workflows/airflow-operator-release-to-pypi.yml +++ b/.github/workflows/airflow-operator-release-to-pypi.yml @@ -17,7 +17,7 @@ jobs: with: version: '3.17.3' repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: make airflow-operator + - run: mage airflowOperator - uses: ./.github/workflows/python-tests with: python-version: '3.8' diff --git a/.github/workflows/airflow-operator.yml b/.github/workflows/airflow-operator.yml index 0985e19939b..1aa0bcf83e3 100644 --- a/.github/workflows/airflow-operator.yml +++ b/.github/workflows/airflow-operator.yml @@ -14,7 +14,6 @@ on: - 'docs/python_airflow_operator.md' - 'docs/python_armada_client.md' - 'internal/jobservice/*' - - 'makefile' - 'pkg/api/*.proto' - 'pkg/api/jobservice/*.proto' - 'scripts/build-airflow-operator.sh' @@ -34,7 +33,6 @@ on: - 'docs/python_airflow_operator.md' - 'docs/python_armada_client.md' - 'internal/jobservice/*' - - 'makefile' - 'pkg/api/*.proto' - 'pkg/api/jobservice/*.proto' - 'scripts/build-airflow-operator.sh' @@ -62,7 +60,7 @@ jobs: with: version: '3.17.3' repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: make airflow-operator + - run: mage airflowOperator - uses: ./.github/workflows/python-tests with: python-version: ${{ matrix.python }} @@ -106,5 +104,5 @@ jobs: with: version: '3.17.3' repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: make tests-e2e-airflow + - run: mage teste2eAirflow diff --git a/.github/workflows/build-release-images.yml b/.github/workflows/build-release-images.yml index 51a335dbea4..d90d6f298ec 100644 --- a/.github/workflows/build-release-images.yml +++ b/.github/workflows/build-release-images.yml @@ -17,12 +17,12 @@ jobs: steps: - uses: actions/checkout@v3.1.0 - uses: ./.github/workflows/go-setup - - run: make build-ci - - run: make tests-e2e-setup - - run: make tests-e2e-no-setup + - run: mage BuildCI + - run: mage teste2eSetup + - run: mage testse2eNoSetup env: INTEGRATION_ENABLED: true - - run: make junit-report + - run: mage junitReport - name: Upload junit report uses: actions/upload-artifact@v3.1.1 with: @@ -33,7 +33,7 @@ jobs: - name: Save docker images to artifact run: | mkdir -p docker-images - docker save armada | gzip > docker-images/armada.tar.gz + docker save armada-server | gzip > docker-images/armada-server.tar.gz docker save armada-executor | gzip > docker-images/armada-executor.tar.gz docker save armadactl | gzip > docker-images/armadactl.tar.gz docker save testsuite | gzip > docker-images/testsuite.tar.gz diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cda45fd1ef3..06f6bb74b98 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -46,8 +46,8 @@ jobs: version: '3.17.3' repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: ./.github/workflows/go-setup - - name: Make Proto - run: make proto + - name: Mage Proto + run: mage proto # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/go-setup/action.yml b/.github/workflows/go-setup/action.yml index 622969b0996..f1bfae19cdd 100644 --- a/.github/workflows/go-setup/action.yml +++ b/.github/workflows/go-setup/action.yml @@ -9,4 +9,4 @@ runs: go-version: ${{ matrix.go }} - name: Setup dependencies shell: bash - run: make download \ No newline at end of file + run: mage download \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml index 7fb7ad36dca..088a8e846b3 100644 --- a/.github/workflows/python-client.yml +++ b/.github/workflows/python-client.yml @@ -87,4 +87,4 @@ jobs: with: version: '3.17.3' repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: make tests-e2e-python + - run: mage teste2epython diff --git a/.github/workflows/python-tests/action.yml b/.github/workflows/python-tests/action.yml index d9f68202c43..d4378bddffe 100644 --- a/.github/workflows/python-tests/action.yml +++ b/.github/workflows/python-tests/action.yml @@ -34,7 +34,7 @@ runs: version: '3.17.3' repo-token: ${{ inputs.github-token }} # Generate the proto files for python, required for later steps - - run: make python + - run: mage buildPython shell: bash - name: Run tox format environment run: tox -e format diff --git a/.goreleaser.yml b/.goreleaser.yml index 034dc8a346b..407598013f9 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -369,8 +369,8 @@ dockers: goos: linux goarch: amd64 image_templates: - - "{{ .Env.DOCKER_REPO }}armada-testsuite:latest" - - "{{ .Env.DOCKER_REPO }}armada-testsuite:{{ .Version }}" + - "{{ .Env.DOCKER_REPO }}testsuite:latest" + - "{{ .Env.DOCKER_REPO }}testsuite:{{ .Version }}" build_flag_templates: *BUILD_FLAG_TEMPLATES ids: - testsuite diff --git a/magefiles/airflow.go b/magefiles/airflow.go index b92140ae0e3..d9f4a2b9afe 100644 --- a/magefiles/airflow.go +++ b/magefiles/airflow.go @@ -72,7 +72,7 @@ func stopAirflow() error { } // AirflowOperator builds the Airflow Operator -func airflowOperator() error { +func AirflowOperator() error { fmt.Println("Building Airflow Operator...") err := os.RemoveAll("proto-airflow") @@ -85,12 +85,12 @@ func airflowOperator() error { return fmt.Errorf("failed to create proto-airflow directory: %w", err) } - _, err = dockerOutput("buildx", "build", "-o", "type=docker", "-t", "armada-airflow-operator-builder", "-f", "./build/airflow-operator/Dockerfile", ".") + err = dockerRun("buildx", "build", "-o", "type=docker", "-t", "armada-airflow-operator-builder", "-f", "./build/airflow-operator/Dockerfile", ".") if err != nil { return fmt.Errorf("failed to build Airflow Operator: %w", err) } - _, err = dockerOutput("run", "--rm", "-v", "${PWD}/proto-airflow:/proto-airflow", "-v", "${PWD}:/go/src/armada", "-w", "/go/src/armada", "armada-airflow-operator-builder", "./scripts/build-airflow-operator.sh") + err = dockerRun("run", "--rm", "-v", "${PWD}/proto-airflow:/proto-airflow", "-v", "${PWD}:/go/src/armada", "-w", "/go/src/armada", "armada-airflow-operator-builder", "./scripts/build-airflow-operator.sh") if err != nil { return fmt.Errorf("failed to run build-airflow-operator.sh script: %w", err) } diff --git a/magefiles/main.go b/magefiles/main.go index 8ca77e7f5e1..a0dfadc5b8b 100644 --- a/magefiles/main.go +++ b/magefiles/main.go @@ -236,7 +236,7 @@ func JunitReport() error { } // Make sure everything has been synced to disk - if err := sh.Run("sync"); err != nil { + if err := sh.RunV("sync"); err != nil { return fmt.Errorf("failed to sync: %w", err) } @@ -252,7 +252,7 @@ func JunitReport() error { } if len(goTestCmd) == 0 { - if err = sh.RunV("bash", "-c", "cat test_reports/*.txt | go-junit-report > test_reports/junit.xml"); err != nil { + if err := sh.RunV("bash", "-c", "cat test_reports/*.txt | go-junit-report > test_reports/junit.xml"); err != nil { return err } } else { @@ -264,7 +264,7 @@ func JunitReport() error { return nil } -// Code generation tasks: statik, goimports, go generate. +// Code generation tasks: statik, goimports, go generate. func Generate() error { go_cmd, err := go_CMD() if err != nil { @@ -310,3 +310,13 @@ func Generate() error { } return nil } + +// CI Image to build +func BuildCI() error { + ciImage := []string{"server", "executor", "armadactl", "testsuite", "lookout", "lookoutingester", "lookoutv2", "lookoutingesterv2", "eventingester", "scheduler", "scheduleringester", "binoculars", "jobservice"} + err := goreleaserMinimalRelease(ciImage...) + if err != nil { + return err + } + return nil +} diff --git a/magefiles/tests.go b/magefiles/tests.go index 63e76b78863..f7e7f7fe7cd 100644 --- a/magefiles/tests.go +++ b/magefiles/tests.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/magefile/mage/mg" @@ -41,6 +42,115 @@ func gotestsum() error { return nil } +// Tests is a mage target that runs the tests and generates coverage reports. +func Tests() error { + mg.Deps(gotestsum) + var err error + + docker_Net, err := dockerNet() + if err != nil { + return err + } + + err = dockerRun("run", "-d", "--name=redis", docker_Net, "-p=6379:6379", "redis:6.2.6") + if err != nil { + return err + } + + err = dockerRun("run", "-d", "--name=postgres", docker_Net, "-p", "5432:5432", "-e", "POSTGRES_PASSWORD=psw", "postgres:14.2") + if err != nil { + return err + } + + err = sh.Run("sleep", "3") + if err != nil { + return err + } + packages, err := sh.Output("go", "list", "./internal/...") + if err != nil { + return err + } + + internalPackages := filterPackages(strings.Fields(packages), "jobservice/repository") + + err = runtest("internal_coverage.xml", "internal.txt", true, internalPackages...) + if err != nil { + return err + } + + os.Setenv("JSDBTYPE", "sqlite") + err = runtest("", "internal.txt", true, "./internal/jobservice/repository/...") + if err != nil { + return err + } + + os.Setenv("JSDBTYPE", "postgres") + err = runtest("", "internal.txt", true, "./internal/jobservice/repository/...") + if err != nil { + return err + } + os.Unsetenv("JSDBTYPE") + err = runtest("pkg_coverage.xml", "pkg.txt", false, "./pkg...") + if err != nil { + return err + } + + err = runtest("cmd_coverage.xml", "cmd.txt", false, "./cmd...") + if err != nil { + return err + } + + defer func() { + dockerErr := dockerRun("rm", "-f", "redis", "postgres") + if dockerErr != nil { + if err == nil { + err = dockerErr + } else { + err = fmt.Errorf("%w; %s", err, dockerErr.Error()) + } + } + }() + + return err +} + +func runtest(coverageFileName, outputFileName string, appendOutput bool, directories ...string) error { + args := []string{"--", "-v"} + if coverageFileName != "" { + args = append(args, "-coverprofile", coverageFileName) + } + args = append(args, directories...) + + cmd := exec.Command(Gotestsum, args...) + + fileFlags := os.O_WRONLY | os.O_CREATE + if appendOutput { + fileFlags |= os.O_APPEND + } else { + fileFlags |= os.O_TRUNC + } + + file, err := os.OpenFile(filepath.Join("test_reports", outputFileName), fileFlags, 0o644) + if err != nil { + return err + } + defer file.Close() + + cmd.Stdout = io.MultiWriter(os.Stdout, file) + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func filterPackages(packages []string, filter string) []string { + var filtered []string + for _, pkg := range packages { + if !strings.Contains(pkg, filter) { + filtered = append(filtered, pkg) + } + } + return filtered +} + func runTest(name, outputFileName string) error { cmd := exec.Command(Gotestsum, "--", "-v", name, "-count=1") file, err := os.Create(filepath.Join("test_reports", outputFileName)) @@ -107,7 +217,7 @@ func Teste2eSetup() error { } mg.Deps(Kind) mg.Deps(StartDependencies) - os.Setenv("ARMADA_COMPONENTS","server,lookoutingester,executor,eventingester,jobservice") + os.Setenv("ARMADA_COMPONENTS", "server,lookoutingester,executor,eventingester,jobservice") mg.Deps(StartComponents) if err := dockerRun("run", "-d", "--name", "lookout-ingester-migrate", "--network=kind", "-v", "${PWD}/e2e:/e2e", @@ -143,26 +253,25 @@ func Teste2eSetup() error { } time.Sleep(10 * time.Second) - // Teardown function - defer func() error { + defer func() { fmt.Println("\nexecutor logs:") if err := dockerRun("logs", "executor"); err != nil { - return err + fmt.Println("Error retrieving executor logs:", err) } + fmt.Println("\nserver logs:") if err := dockerRun("logs", "server"); err != nil { - return err + fmt.Println("Error retrieving server logs:", err) } LocalDevStop() - if err = sh.Run("rm", ".kube/config"); err != nil { - return err + if err := sh.Run("rm", ".kube/config"); err != nil { + fmt.Println("Error removing .kube/config:", err) } - if err = sh.Run("rmdir", ".kube"); err != nil { - return err + if err := sh.Run("rmdir", ".kube"); err != nil { + fmt.Println("Error removing .kube directory:", err) } - return nil }() if err := runTest("./e2e/armadactl_test/...", "e2e_armadactl.txt"); err != nil { @@ -191,7 +300,7 @@ func Teste2eSetup() error { // Teste2eAirflow runs e2e tests for airflow func Teste2eAirflow() error { - mg.Deps(airflowOperator) + mg.Deps(AirflowOperator) if err := BuildDockers("jobservice"); err != nil { return err } @@ -226,7 +335,7 @@ func Teste2eAirflow() error { fmt.Println(err) } - _, err = dockerOutput("rm", "-f", "jobservice") + err = dockerRun("rm", "-f", "jobservice") if err != nil { fmt.Println(err) }