Merge branch 'master' into snyk-upgrade-129641cf3784925ff310b2748dbbb57a

armadaproject · Aug 24, 2023 · 1f06780 · 1f06780
2 parents 0e3f2d7 + 15ed4ee
commit 1f06780
Show file tree

Hide file tree

Showing 247 changed files with 8,736 additions and 2,078 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.github/workflows/airflow-operator-release-to-pypi.yml b/.github/workflows/airflow-operator-release-to-pypi.yml
@@ -13,11 +13,11 @@ jobs:
       - uses: actions/checkout@v3.3.0
       - uses: ./.github/workflows/go-setup
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v2
         with:
-          version: '3.17.3'
+          version: '23.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make airflow-operator
+      - run: go run github.com/magefile/mage@v1.14.0 -v airflowOperator
       - uses: ./.github/workflows/python-tests
         with:
           python-version: '3.8'

diff --git a/.github/workflows/airflow-operator.yml b/.github/workflows/airflow-operator.yml
@@ -14,7 +14,6 @@ on:
       - 'docs/python_airflow_operator.md'
       - 'docs/python_armada_client.md'
       - 'internal/jobservice/*'
-      - 'makefile'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
       - 'scripts/build-airflow-operator.sh'
@@ -34,7 +33,6 @@ on:
       - 'docs/python_airflow_operator.md'
       - 'docs/python_armada_client.md'
       - 'internal/jobservice/*'
-      - 'makefile'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
       - 'scripts/build-airflow-operator.sh'
@@ -58,11 +56,11 @@ jobs:
       - uses: actions/checkout@v3.3.0
       - uses: ./.github/workflows/go-setup
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v2
         with:
-          version: '3.17.3'
+          version: '23.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make airflow-operator
+      - run: go run github.com/magefile/mage@v1.14.0 -v airflowOperator
       - uses: ./.github/workflows/python-tests
         with:
           python-version: ${{ matrix.python }}
@@ -102,9 +100,9 @@ jobs:
           mkdir -p .kube/external
           go run github.com/magefile/mage@v1.14.0 -v localdev minimal
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v2
         with:
-          version: '3.17.3'
+          version: '23.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make tests-e2e-airflow
+      - run: go run github.com/magefile/mage@v1.14.0 -v teste2eAirflow
 
diff --git a/.github/workflows/autoupdate.yml b/.github/workflows/autoupdate.yml
@@ -0,0 +1,22 @@
+name: autoupdate
+on:
+  # This will trigger on all pushes to all branches.
+  push: {}
+  # Alternatively, you can only trigger if commits are pushed to certain branches, e.g.:
+  # push:
+  #   branches:
+  #     - master
+  #     - unstable
+jobs:
+  autoupdate:
+    name: autoupdate
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: docker://chinthakagodawita/autoupdate-action:v1
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+	  PR_LABELS: "auto-update"
+          MERGE_MSG: "Branch was auto-updated."
+          RETRY_COUNT: "5"
+          RETRY_SLEEP: "300"
+          MERGE_CONFLICT_ACTION: "fail"
diff --git a/.github/workflows/build-release-images.yml b/.github/workflows/build-release-images.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -35,12 +35,18 @@ jobs:
   # Virtual job that can be configured as a required check before a PR can be merged.
   all-required-checks-done:
     name: All required checks done
+    if: ${{ always() }}
     needs:
       - lint
-      - codeql
       - test
     runs-on: ubuntu-22.04
     steps:
-      - run: |
-          echo "All required checks done"
-
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            const results = ${{ toJSON(needs.*.result) }};
+            if (results.every(res => res === 'success')) {
+              core.info('All required checks succeeded');
+            } else {
+              core.setFailed('Some required checks failed');
+            }
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -41,13 +41,13 @@ jobs:
       uses: actions/checkout@v3.3.0
     # The ArmadaProject.Io.Client needs the generated proto files
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@v2
       with:
-        version: '3.17.3'
+        version: '23.3'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - uses: ./.github/workflows/go-setup
-    - name: Make Proto
-      run: make proto
+    - name: Mage Proto
+      run: go run github.com/magefile/mage@v1.14.0 -v proto
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/go-setup/action.yml b/.github/workflows/go-setup/action.yml
@@ -9,4 +9,5 @@ runs:
         go-version: ${{ matrix.go }}
     - name: Setup dependencies
       shell: bash
-      run: make download
+      run: go run github.com/magefile/mage@v1.14.0 download
+
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -26,14 +26,29 @@ jobs:
 
       - name: Check TypeScript formatting
         run: |
-          yarn install --frozen-lockfile && yarn run fmt || exit 1
+          yarn install --frozen-lockfile && yarn run fmt || true
           exit $(git status -s -uno | wc -l)
         working-directory: ./internal/lookout/ui
+        continue-on-error: true  
+
+      - name: Generating TypeScript lint results as summary
+        working-directory: ./internal/lookout/ui
+        if: ${{ always() }}
+        run: |
+           yarn run lint > lint_results.txt || true
+           lint_results=$(cat lint_results.txt)
+           echo -e "## 🪧 Typescript Lint Results\n" >> $GITHUB_STEP_SUMMARY
+           if [[ $lint_results =~ "problem" ]]; then
+              echo -e "### List of Lint Issues \n" >> $GITHUB_STEP_SUMMARY
+              echo -e "${lint_results}" >> $GITHUB_STEP_SUMMARY
+           else
+              echo -e "### No Lint issues found.\n" >> $GITHUB_STEP_SUMMARY   
+           fi   
+        continue-on-error: true
 
   go-lint:
     name: Lint Go
     runs-on: ubuntu-22.04
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3.3.0
@@ -51,3 +66,17 @@ jobs:
           version: v1.52.2
           only-new-issues: true
           args: --timeout=10m --issues-exit-code=1 --sort-results ./...
+
+      - name: Generating Golang lint results as summary
+        if: ${{ always() }}
+        run: |
+          golangci-lint run > lint_results.txt || true
+          lint_results=$(cat lint_results.txt)
+          echo -e "## 🪧 Go Lint Results\n" >> $GITHUB_STEP_SUMMARY
+          if [ -z "$lint_results" ]; then
+            echo -e "### No Lint Issues found.\n" >> $GITHUB_STEP_SUMMARY
+          else
+              echo -e "###  List of Lint Issues \n" >> $GITHUB_STEP_SUMMARY
+              echo -e "${lint_results}" >> $GITHUB_STEP_SUMMARY
+          fi
+        continue-on-error: true
diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
@@ -32,6 +32,7 @@ jobs:
           cp master/docs/*.md gh-pages/
           rm gh-pages/docs-readme.md
           cp -r master/docs/quickstart gh-pages/
+          cp master/docs/quickstart/index.md gh-pages/quickstart.md
           cp -r master/docs/developer gh-pages/
           cp -r master/docs/design gh-pages/
           cp master/CODE_OF_CONDUCT.md master/CONTRIBUTING.md gh-pages/_pages/

diff --git a/.github/workflows/python-client-release-to-pypi.yml b/.github/workflows/python-client-release-to-pypi.yml
@@ -13,9 +13,9 @@ jobs:
       - uses: actions/checkout@v3.3.0
       - uses: ./.github/workflows/go-setup
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v2
         with:
-          version: '3.17.3'
+          version: '23.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: ./.github/workflows/python-tests
         with:

diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml
@@ -83,8 +83,8 @@ jobs:
           mkdir -p .kube/external
           go run github.com/magefile/mage@v1.14.0 -v localdev minimal
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v2
         with:
-          version: '3.17.3'
+          version: '23.3'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make tests-e2e-python
+      - run: go run github.com/magefile/mage@v1.14.0 -v teste2epython
diff --git a/.github/workflows/python-tests/action.yml b/.github/workflows/python-tests/action.yml
@@ -5,36 +5,31 @@ inputs:
   path:
     description: "Path to python package root relative to repo root"
     required: true
-    type: string
   python-version:
     description: "Version of python to setup and run tests against"
     required: true
-    type: string
   tox-env:
     description: "Tox environment to use for running the tests"
     required: true
-    type: string
   github-token:
     description: "Token for authenticated github requests"
     required: true
-    type: string
-
 runs:
   using: "composite"
   steps:
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ inputs.python-version }}
     # Tox to run tests; build to build the wheel after tests pass
     - run: pip install tox==3.27.1 build twine
       shell: bash
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@v2
       with:
-        version: '3.17.3'
+        version: '23.3'
         repo-token: ${{ inputs.github-token }}
     # Generate the proto files for python, required for later steps
-    - run: make python
+    - run: go run github.com/magefile/mage@v1.14.0 -v buildPython
       shell: bash
     - name: Run tox format environment
       run: tox -e format

diff --git a/.github/workflows/slack-alerts.yml b/.github/workflows/slack-alerts.yml
@@ -2,38 +2,18 @@ name: Slack CI Alerts
 
 on:
   workflow_run:
-    workflows: [CI, Python Airflow Operator, Build Release Images]
+    workflows: [CI, Python Airflow Operator, Build Release Images, Release Armada components, Release Armada components - RC]
     types: [completed]
 
 jobs:
-  on_push_failure:
+  on-failure:
     runs-on: ubuntu-latest
-    if: github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'master'
-    steps:
-      - name: "Master Notification"
-        uses: Mo-Fatah/ci-alerts@v1
-        env: 
-          webhook: ${{ secrets.SLACK_WEBHOOK }}
-          event: push
-          commit: ${{ github.sha }}
-          commit_url: https://github.com/armadaproject/armada/commit/${{ github.sha }}
-          author: ${{ github.actor }}
-          workflow_name: ${{ github.event.workflow_run.name }}
-          workflow_url: ${{ github.event.workflow_run.html_url}}
-
-  on_pull_request_failure:
-    runs-on: ubuntu-latest
-    if: github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'pull_request'
+    if: github.event.workflow_run.conclusion == 'failure' 
     steps:
       - uses: actions/checkout@v3.3.0
-      - name: "Pull Request Notification"
-        uses: Mo-Fatah/ci-alerts@v1
+      - name: "Send Notification"
+        uses: Mo-Fatah/ci-alerts@v2
         env: 
           webhook: ${{ secrets.SLACK_WEBHOOK }}
-          event: pr
-          commit: ${{ github.sha }}
-          commit_url: https://github.com/armadaproject/armada/commit/${{ github.sha }}
-          author: ${{ github.actor }}
-          workflow_name: ${{ github.event.workflow_run.name }}
-          workflow_url: ${{ github.event.workflow_run.html_url}}
-          users_path: ${{github.workspace}}/.github/gh-to-slackid
+          github_context: ${{ toJSON(github) }}
+          users_path: ${{github.workspace}}/.github/gh-to-slackid