Merge branch 'master' into add-badge-lfx-insights

armadaproject · Jun 11, 2024 · 7b00ed8 · 7b00ed8
2 parents f6b5472 + 8b38477
commit 7b00ed8
Show file tree

Hide file tree

Showing 437 changed files with 16,051 additions and 24,528 deletions.
diff --git a/.devcontainer/demo/README.md b/.devcontainer/demo/README.md
@@ -5,7 +5,7 @@ Run this when the script completes
 ```bash
 # If you are at the root of the repo
 cd .devcontainer/demo
-./armadactl create queue test --priorityFactor 1
+./armadactl create queue test --priority-factor 1
 ./armadactl submit jobs.yaml
 ./armadactl watch test job-set-1
 ```
@@ -21,4 +21,4 @@ Forward these ports:
 - 10000: Lookoutv2 API
 - 8080: Armada Server API
 
-and go to: http://localhost:8089
+and go to: http://localhost:8089
diff --git a/.github/actions/setup-go-cache/action.yml b/.github/actions/setup-go-cache/action.yml
@@ -5,9 +5,9 @@ inputs:
     description: "Prefix for the cache key"
     required: true
   go-version:
-    description: "Version of Go. Default 1.20"
+    description: "Version of Go. Default 1.21"
     required: false
-    default: "1.20"
+    default: "1.21"
   cache-tools:
     description: "True/false flag to cache tools"
     required: false

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,7 +24,7 @@ jobs:
       - uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
-          version: v1.20.0
+          version: v1.24.0
           args: release  --snapshot --skip-sbom --skip-sign --clean
         env:
           DOCKER_REPO: "gresearch"
@@ -88,5 +88,4 @@ jobs:
         with:
           name: nupkg-artifacts
           path: |
-            ./bin/client/DotNet/G-Research.Armada.Client.${{ steps.create-release-tag.outputs.release_tag }}.nupkg
             ./bin/client/DotNet/ArmadaProject.Io.Client.${{ steps.create-release-tag.outputs.release_tag }}.nupkg
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -83,7 +83,7 @@ jobs:
         uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
-          version: v1.19.2
+          version: v1.24.0
           args: "-f ./.goreleaser.yml release --clean"
         env:
           FULL_RELEASE: true
@@ -119,4 +119,4 @@ jobs:
           TAG: ${{ github.event.workflow_run.head_branch }}
         run: |
           VERSION=${TAG#v}
-          dotnet nuget push ./bin/client/DotNet/G-Research.Armada.Client.$VERSION.nupkg ./bin/client/DotNet/ArmadaProject.Io.Client.$VERSION.nupkg -k ${{ secrets.NUGET_API_KEY }} -s https://api.nuget.org/v3/index.json
+          dotnet nuget push ./bin/client/DotNet/ArmadaProject.Io.Client.$VERSION.nupkg -k ${{ secrets.NUGET_API_KEY }} -s https://api.nuget.org/v3/index.json
diff --git a/.gitignore b/.gitignore
@@ -70,7 +70,6 @@ msbuild.log
 msbuild.err
 msbuild.wrn
 packages
-client/DotNet.gRPC/Armada.Client.Grpc/generated
 
 # python
 # Pycache, dist, poetry versions
@@ -94,6 +93,7 @@ build/ssl/certs/ca-certificates.crt
 developer/volumes/go/pkg/
 .coverage
 coverage.xml
+.venv
 
 # Yarn
 developer/yarn.lock

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -19,7 +19,7 @@ env:
   # To use a builder other than "default", set this variable.
   # Necessary for, e.g., GitHub actions cache integration.
   - DOCKER_BUILDX_BUILDER={{ if index .Env "DOCKER_BUILDX_BUILDER"  }}{{ .Env.DOCKER_BUILDX_BUILDER }}{{ else }}default{{ end }}
-  - GOVERSION={{ if index .Env "GOVERSION"  }}{{ .Env.GOVERSION }}{{ else }}go1.20{{ end }}
+  - GOVERSION={{ if index .Env "GOVERSION"  }}{{ .Env.GOVERSION }}{{ else }}go1.21{{ end }}
 
 builds:
   - env: [CGO_ENABLED=0]
@@ -49,15 +49,6 @@ builds:
       - linux
     goarch:
       - amd64
-  - env: [CGO_ENABLED=0]
-    id: pulsartest
-    binary: pulsartest
-    main: ./cmd/pulsartest/main.go
-    mod_timestamp: '{{ .CommitTimestamp }}'
-    goos:
-      - linux
-    goarch:
-      - amd64
   - env: [CGO_ENABLED=0]
     id: testsuite
     binary: testsuite

diff --git a/.run/Armada (Pulsar Scheduler).run.xml → .run/Armada.run.xml b/.run/Armada (Pulsar Scheduler).run.xml → .run/Armada.run.xml
@@ -1,11 +1,11 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Armada (Pulsar Scheduler)" type="CompoundRunConfigurationType">
+  <configuration default="false" name="Armada" type="CompoundRunConfigurationType">
     <toRun name="Armada Infrastructure Services" type="docker-deploy" />
     <toRun name="Event Ingester" type="GoApplicationRunConfiguration" />
     <toRun name="Lookout Ingester V2" type="GoApplicationRunConfiguration" />
     <toRun name="LookoutV2" type="GoApplicationRunConfiguration" />
-    <toRun name="Pulsar Executor" type="GoApplicationRunConfiguration" />
-    <toRun name="Pulsar Server" type="GoApplicationRunConfiguration" />
+    <toRun name="Executor" type="GoApplicationRunConfiguration" />
+    <toRun name="Server" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler Ingester" type="GoApplicationRunConfiguration" />
     <toRun name="postgresPulsarMigration" type="GoApplicationRunConfiguration" />
@@ -15,10 +15,10 @@
     <toRun name="Event Ingester" type="GoApplicationRunConfiguration" />
     <toRun name="Lookout Ingester V2" type="GoApplicationRunConfiguration" />
     <toRun name="LookoutV2" type="GoApplicationRunConfiguration" />
-    <toRun name="Pulsar Executor" type="GoApplicationRunConfiguration" />
-    <toRun name="Pulsar Server" type="GoApplicationRunConfiguration" />
+    <toRun name="Executor" type="GoApplicationRunConfiguration" />
+    <toRun name="Server" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler Ingester" type="GoApplicationRunConfiguration" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Pulsar Executor.run.xml → .run/Executor.run.xml b/.run/Pulsar Executor.run.xml → .run/Executor.run.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Pulsar Executor" type="GoApplicationRunConfiguration" factoryName="Go Application">
+  <configuration default="false" name="Executor" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>

diff --git a/.run/Legacy Server.run.xml b/.run/Legacy Server.run.xml
diff --git a/.run/Lookout Ingester V2.run.xml b/.run/Lookout Ingester V2.run.xml
@@ -4,7 +4,6 @@
     <working_directory value="$PROJECT_DIR$" />
     <envs>
       <env name="ARMADA_METRICS_PORT" value="9006" />
-      <env name="ARMADA_POSTGRES_CONNECTION_DBNAME" value="postgresv2" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
@@ -15,4 +14,4 @@
     <filePath value="$PROJECT_DIR$/cmd/lookoutingesterv2/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/LookoutV2.run.xml b/.run/LookoutV2.run.xml
@@ -4,7 +4,6 @@
     <working_directory value="$PROJECT_DIR$" />
     <envs>
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
-      <env name="ARMADA_POSTGRES_CONNECTION_DBNAME" value="postgresv2" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
     </envs>
@@ -14,4 +13,4 @@
     <filePath value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Run Migrations.run.xml b/.run/Run Migrations.run.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Migrations" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="mage runMigrations" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/usr/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="false" />
+    <envs />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/.run/Scheduler.run.xml b/.run/Scheduler.run.xml
@@ -4,16 +4,16 @@
     <working_directory value="$PROJECT_DIR$" />
     <parameters value="run --config $PROJECT_DIR$/developer/config/insecure-armada.yaml" />
     <envs>
+      <env name="ARMADA_ARMADAAPI_ARMADAURL" value="localhost:50051" />
       <env name="ARMADA_HTTP_PORT" value="8084" />
       <env name="ARMADA_METRICS_PORT" value="9004" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
-      <env name="ARMADA_REDIS_ADDRS" value="localhost:6379" />
     </envs>
     <kind value="FILE" />
     <package value="github.com/armadaproject/armada" />
     <directory value="$PROJECT_DIR$" />
     <filePath value="$PROJECT_DIR$/cmd/scheduler/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Pulsar Server.run.xml → .run/Server.run.xml b/.run/Pulsar Server.run.xml → .run/Server.run.xml
@@ -1,15 +1,15 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Pulsar Server" type="GoApplicationRunConfiguration" factoryName="Go Application">
+  <configuration default="false" name="Server" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <parameters value="--config ./developer/config/insecure-armada.yaml" />
     <envs>
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_EVENTSAPIREDIS_ADDRS" value="localhost:6379" />
       <env name="ARMADA_METRICSPORT" value="9005" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
-      <env name="ARMADA_REDIS_ADDRS" value="localhost:6379" />
       <env name="EXECUTOR_UPDATE_INTERVAL" value="&quot;1s&quot;" />
     </envs>
     <kind value="FILE" />

diff --git a/.run/Start Dependencies.run.xml b/.run/Start Dependencies.run.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Start Dependencies" type="ShConfigurationType">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/scripts/infrastructure-start.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/usr/bin/zsh" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/.run/lookoutv2PostgresMigration.run.xml b/.run/lookoutv2PostgresMigration.run.xml
@@ -5,7 +5,6 @@
     <parameters value="--migrateDatabase" />
     <envs>
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
-      <env name="ARMADA_POSTGRES_CONNECTION_DBNAME" value="postgresv2" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
     </envs>

diff --git a/README.md b/README.md
@@ -46,6 +46,7 @@ For documentation, see the following:
 - [User guide](./docs/user.md)
 - [Quickstart](./docs/quickstart/index.md)
 - [Development guide](./docs/developer.md)
+- [Release notes/Version history](https://github.com/armadaproject/armada/releases)
 - [API Documentation](./docs/developer/api.md)
 
 We expect readers of the documentation to have a basic understanding of Docker and Kubernetes; see, e.g., the following links:

diff --git a/build/airflow-operator/Dockerfile b/build/airflow-operator/Dockerfile
@@ -1,6 +1,6 @@
-ARG PYTHON_VERSION=3.8.15
-
-FROM --platform=x86_64 python:${PYTHON_VERSION}-buster
+ARG PLATFORM=x86_64
+ARG BASE_IMAGE=python:3.8.18-bookworm
+FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto
 

diff --git a/build/python-client/Dockerfile b/build/python-client/Dockerfile
@@ -1,6 +1,7 @@
-ARG PYTHON_VERSION=3.8.18
+ARG PLATFORM=x86_64
+ARG BASE_IMAGE=python:3.8.18-bookworm
 
-FROM --platform=x86_64 python:${PYTHON_VERSION}-bookworm
+FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto
 

diff --git a/client/DotNet/Armada.Client.Test/Armada.Client.Test.csproj b/client/DotNet/Armada.Client.Test/Armada.Client.Test.csproj