Mage conversion (#2652)

* converting make commands to mage Signed-off-by: ntishchauhan0022 <nitishchauhan0022@gmail.com> * resolving errors Signed-off-by: ntishchauhan0022 <nitishchauhan0022@gmail.com> * adding teardown target Signed-off-by: ntishchauhan0022 <nitishchauhan0022@gmail.com> --------- Signed-off-by: ntishchauhan0022 <nitishchauhan0022@gmail.com>
armadaproject · Jul 26, 2023 · b57c175 · b57c175
1 parent d7e5a05
commit b57c175
Show file tree

Hide file tree

Showing 18 changed files with 1,138 additions and 27 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -160,33 +160,33 @@ jobs:
             curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
             chmod +x kubectl
             mv kubectl /home/circleci/bin/
-            make download # no-op if we restored from cache
+            go run github.com/magefile/mage@v1.14.0 -v download
 
       - run:
           name: Build
           command: |
-            make build-ci
-            make build-docker-full-bundle
+            go run github.com/magefile/mage@v1.14.0 -v buildCI
+            go run github.com/magefile/mage@v1.14.0 -v buildDockers full-bundle
 
       - run:
           name: e2e test environment setup
-          command: make tests-e2e-setup
+          command: go run github.com/magefile/mage@v1.14.0 -v teste2eSetup
 
       - run:
           name: Enable integration tests
           command: echo 'export INTEGRATION_ENABLED="true"' >> $BASH_ENV
 
       - run:
           name: e2e tests
-          command: make tests-e2e-no-setup
+          command: go run github.com/magefile/mage@v1.14.0 -v testse2eNoSetup
 
       - run:
           name: e2e test environment teardown
-          command: make tests-e2e-teardown
+          command: go run github.com/magefile/mage@v1.14.0 -v testse2eTearDown
 
       - run:
           name: Prepare reports
-          command: make junit-report
+          command: go run github.com/magefile/mage@v1.14.0 -v junitReport
           when: always
 
       - store_test_results:
@@ -288,7 +288,7 @@ jobs:
             curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
             chmod +x kubectl
             sudo mv kubectl /usr/local/bin/
-            make download # no-op if we restored from cache
+            go run github.com/magefile/mage@v1.14.0 -v download
 
       - run:
           name: Build armadactl release artifacts
@@ -429,7 +429,7 @@ jobs:
             curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
             chmod +x kubectl
             sudo mv kubectl /usr/local/bin/
-            make download # no-op if we restored from cache
+            go run github.com/magefile/mage@v1.14.0 -v download # no-op if we restored from cache
 
       - run:
           name: Push dotnet clients to nuget
@@ -440,7 +440,7 @@ jobs:
             go mod download
             go run github.com/magefile/mage@v1.14.0 -v BootstrapTools
             go run github.com/magefile/mage@v1.14.0 -v proto
-            RELEASE_TAG=${CIRCLE_TAG#"v"} make push-nuget
+            RELEASE_TAG=${CIRCLE_TAG#"v"} go run github.com/magefile/mage@v1.14.0 -v pushNuget
 
       - store_artifacts:
           path: bin/client/DotNet

diff --git a/.github/workflows/airflow-operator-release-to-pypi.yml b/.github/workflows/airflow-operator-release-to-pypi.yml
@@ -17,7 +17,7 @@ jobs:
         with:
           version: '3.17.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make airflow-operator
+      - run: go run github.com/magefile/mage@v1.14.0 -v airflowOperator
       - uses: ./.github/workflows/python-tests
         with:
           python-version: '3.8'

diff --git a/.github/workflows/airflow-operator.yml b/.github/workflows/airflow-operator.yml
@@ -14,7 +14,6 @@ on:
       - 'docs/python_airflow_operator.md'
       - 'docs/python_armada_client.md'
       - 'internal/jobservice/*'
-      - 'makefile'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
       - 'scripts/build-airflow-operator.sh'
@@ -34,7 +33,6 @@ on:
       - 'docs/python_airflow_operator.md'
       - 'docs/python_armada_client.md'
       - 'internal/jobservice/*'
-      - 'makefile'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
       - 'scripts/build-airflow-operator.sh'
@@ -62,7 +60,7 @@ jobs:
         with:
           version: '3.17.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make airflow-operator
+      - run: go run github.com/magefile/mage@v1.14.0 -v airflowOperator
       - uses: ./.github/workflows/python-tests
         with:
           python-version: ${{ matrix.python }}
@@ -106,5 +104,5 @@ jobs:
         with:
           version: '3.17.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make tests-e2e-airflow
+      - run: go run github.com/magefile/mage@v1.14.0 -v teste2eAirflow
 
diff --git a/.github/workflows/build-release-images.yml b/.github/workflows/build-release-images.yml
@@ -17,12 +17,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3.3.0
       - uses: ./.github/workflows/go-setup
-      - run: make build-ci
-      - run: make tests-e2e-setup
-      - run: make tests-e2e-no-setup
+      - run: go run github.com/magefile/mage@v1.14.0 -v BuildCI
+      - run: go run github.com/magefile/mage@v1.14.0 -v teste2eSetup
+      - run: go run github.com/magefile/mage@v1.14.0 -v testse2eNoSetup
         env:
           INTEGRATION_ENABLED: true
-      - run: make junit-report
+      - run: go run github.com/magefile/mage@v1.14.0 -v junitReport
       - name: Upload junit report
         uses: actions/upload-artifact@v3.1.1
         with:
@@ -33,7 +33,7 @@ jobs:
       - name: Save docker images to artifact
         run: |
           mkdir -p docker-images
-          docker save armada | gzip > docker-images/armada.tar.gz
+          docker save armada-server | gzip > docker-images/armada-server.tar.gz 
           docker save armada-executor | gzip > docker-images/armada-executor.tar.gz
           docker save armadactl | gzip > docker-images/armadactl.tar.gz
           docker save testsuite | gzip > docker-images/testsuite.tar.gz

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -46,8 +46,8 @@ jobs:
         version: '3.17.3'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - uses: ./.github/workflows/go-setup
-    - name: Make Proto
-      run: make proto
+    - name: Mage Proto
+      run: go run github.com/magefile/mage@v1.14.0 -v proto
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/go-setup/action.yml b/.github/workflows/go-setup/action.yml
@@ -9,4 +9,5 @@ runs:
         go-version: ${{ matrix.go }}
     - name: Setup dependencies
       shell: bash
-      run: make download
+      run: go run github.com/magefile/mage@v1.14.0 download
+
diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml
@@ -87,4 +87,4 @@ jobs:
         with:
           version: '3.17.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make tests-e2e-python
+      - run: go run github.com/magefile/mage@v1.14.0 -v teste2epython
diff --git a/.github/workflows/python-tests/action.yml b/.github/workflows/python-tests/action.yml
@@ -22,6 +22,9 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - uses: actions/setup-go@v4
+      with:
+        go-version: ${{ matrix.go }}
     - uses: actions/setup-python@v2
       with:
         python-version: ${{ inputs.python-version }}
@@ -34,7 +37,7 @@ runs:
         version: '3.17.3'
         repo-token: ${{ inputs.github-token }}
     # Generate the proto files for python, required for later steps
-    - run: make python
+    - run: go run github.com/magefile/mage@v1.14.0 -v buildPython
       shell: bash
     - name: Run tox format environment
       run: tox -e format

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -46,6 +46,33 @@ builds:
       - linux
     goarch:
       - amd64
+  - env: [CGO_ENABLED=0]
+    id: armadaloadtester
+    binary: armada-load-tester
+    main: ./cmd/armada-load-tester/main.go
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    goos:
+      - linux
+    goarch:
+      - amd64
+  - env: [CGO_ENABLED=0]
+    id: pulsartest
+    binary: pulsartest
+    main: ./cmd/pulsartest/main.go
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    goos:
+      - linux
+    goarch:
+      - amd64
+  - env: [CGO_ENABLED=0]
+    id: testsuite
+    binary: testsuite
+    main: ./cmd/testsuite/main.go
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    goos:
+      - linux
+    goarch:
+      - amd64
   - env: [CGO_ENABLED=0]
     id: binoculars
     binary: binoculars
@@ -329,6 +356,30 @@ dockers:
       - config/executor/config.yaml
     dockerfile: ./build_goreleaser/fakeexecutor/Dockerfile
 
+  - id: armadaloadtester
+    use: buildx
+    goos: linux
+    goarch: amd64
+    image_templates:
+      - "{{ .Env.DOCKER_REPO }}armada-loadtester:latest"
+      - "{{ .Env.DOCKER_REPO }}armada-loadtester:{{ .Version }}"
+    build_flag_templates: *BUILD_FLAG_TEMPLATES
+    ids:
+      - armadaloadtester
+    dockerfile: ./build_goreleaser/loadtester/Dockerfile
+
+  - id: testsuite
+    use: buildx
+    goos: linux
+    goarch: amd64
+    image_templates:
+      - "{{ .Env.DOCKER_REPO }}testsuite:latest"
+      - "{{ .Env.DOCKER_REPO }}testsuite:{{ .Version }}"
+    build_flag_templates: *BUILD_FLAG_TEMPLATES
+    ids:
+      - testsuite
+    dockerfile: ./build_goreleaser/testsuite/Dockerfile
+
   - id: lookoutingester
     use: buildx
     goos: linux

diff --git a/build_goreleaser/loadtester/Dockerfile b/build_goreleaser/loadtester/Dockerfile
@@ -0,0 +1,15 @@
+ARG BASE_IMAGE=alpine:3.17.0
+
+FROM ${BASE_IMAGE}
+LABEL org.opencontainers.image.title=loadtester
+LABEL org.opencontainers.image.description="Load Tester"
+LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/loadtester
+
+RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
+USER armada
+
+COPY armada-load-tester /app/
+
+WORKDIR /app
+
+ENTRYPOINT ["./armada-load-tester"]
diff --git a/magefiles/airflow.go b/magefiles/airflow.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/magefile/mage/sh"
 )
@@ -69,3 +70,30 @@ func stopAirflow() error {
 
 	return sh.Run("rm", "-rf", "localdev/airflow/opt/")
 }
+
+// AirflowOperator builds the Airflow Operator
+func AirflowOperator() error {
+	fmt.Println("Building Airflow Operator...")
+
+	err := os.RemoveAll("proto-airflow")
+	if err != nil {
+		return fmt.Errorf("failed to remove proto-airflow directory: %w", err)
+	}
+
+	err = os.MkdirAll("proto-airflow", os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("failed to create proto-airflow directory: %w", err)
+	}
+
+	err = dockerRun("buildx", "build", "-o", "type=docker", "-t", "armada-airflow-operator-builder", "-f", "./build/airflow-operator/Dockerfile", ".")
+	if err != nil {
+		return fmt.Errorf("failed to build Airflow Operator: %w", err)
+	}
+
+	err = dockerRun("run", "--rm", "-v", "${PWD}/proto-airflow:/proto-airflow", "-v", "${PWD}:/go/src/armada", "-w", "/go/src/armada", "armada-airflow-operator-builder", "./scripts/build-airflow-operator.sh")
+	if err != nil {
+		return fmt.Errorf("failed to run build-airflow-operator.sh script: %w", err)
+	}
+
+	return nil
+}