Merge branch 'master' into flaky

.github/workflows/test.yml

@@ -46,15 +46,6 @@ jobs:
           path: ./internal/lookout/ui/junit.xml
           if-no-files-found: error
 
-      - name: Send Coverage Report to Codecov
-        if: always()
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./internal/lookout/ui/coverage/cobertura-coverage.xml
-          flags: unittests
-          name: codecov-armada-ts-unit-tests
-          verbose: true
-
   go-unit-tests:
     name: Golang Unit Tests
     runs-on: ubuntu-22.04
@@ -94,15 +85,6 @@ jobs:
           path: test-reports/
           if-no-files-found: error
 
-      - name: Send Coverage Report to Codecov
-        if: always()
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./test-reports/coverage.out
-          flags: unittests
-          name: codecov-armada-go-unit-tests
-          verbose: true
-
   go-integration-tests:
     name: Golang Integration Tests
     runs-on: ubuntu-22.04

cmd/eventsprinter/logic/logic.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	v1 "k8s.io/api/core/v1"
 
+	"github.com/armadaproject/armada/internal/common/util"
 	"github.com/armadaproject/armada/pkg/armadaevents"
 )
 
@@ -31,7 +32,15 @@ func PrintEvents(url, topic, subscription string, verbose bool) error {
 				time.Sleep(time.Second)
 				continue
 			}
-			consumer.Ack(msg)
+
+			util.RetryUntilSuccess(
+				ctx,
+				func() error { return consumer.Ack(msg) },
+				func(err error) {
+					fmt.Println(err)
+					time.Sleep(time.Second)
+				},
+			)
 
 			sequence := &armadaevents.EventSequence{}
 			err = proto.Unmarshal(msg.Payload(), sequence)

developer/airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.6.0-python3.10
+FROM apache/airflow:2.7.0-python3.10
 
 RUN umask 0002; \
     mkdir -p /home/airflow/client

developer/airflow/docker-compose.yaml

@@ -24,7 +24,7 @@
 # The following variables are supported:
 #
 # AIRFLOW_IMAGE_NAME           - Docker image name used to run Airflow.
-#                                Default: apache/airflow:2.6.0
+#                                Default: apache/airflow:|version|
 # AIRFLOW_UID                  - User ID in Airflow containers
 #                                Default: 50000
 # AIRFLOW_PROJ_DIR             - Base path to which all the files will be volumed.
@@ -50,7 +50,7 @@ x-airflow-common:
   # In order to add custom dependencies or upgrade provider packages you can use your extended image.
   # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
   # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
-  # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.6.0}
+  # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|}
   build: .
   environment:
     &airflow-common-env
@@ -78,6 +78,7 @@ x-airflow-common:
     - ${AIRFLOW_PROJ_DIR:-.}/armada_client:/opt/airflow/armada_client
     - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
     - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
+    - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config
     - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
   user: "${AIRFLOW_UID:-50000}:0"
   depends_on:
@@ -160,7 +161,7 @@ services:
     healthcheck:
       test:
         - "CMD-SHELL"
-        - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
+        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
       interval: 30s
       timeout: 10s
       retries: 5
@@ -264,7 +265,7 @@ services:
     # yamllint enable rule:line-length
     environment:
       <<: *airflow-common-env
-      _AIRFLOW_DB_UPGRADE: 'true'
+      _AIRFLOW_DB_MIGRATE: 'true'
       _AIRFLOW_WWW_USER_CREATE: 'true'
       _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
       _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}

e2e/pulsar_test/pulsar_test.go

@@ -900,7 +900,11 @@ func receiveJobSetSequencesWithEventFilter(
 			fmt.Println("Pulsar receive error", err)
 			continue
 		}
-		consumer.Ack(msg)
+		err = consumer.Ack(msg)
+		if err != nil {
+			fmt.Println("Pulsar ack error", err)
+			continue
+		}
 
 		sequence := &armadaevents.EventSequence{}
 		err = proto.Unmarshal(msg.Payload(), sequence)

go.mod

@@ -10,7 +10,7 @@ replace github.com/AthenZ/athenz v1.10.39 => github.com/AthenZ/athenz v1.10.4
 require (
 	github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5
 	github.com/alicebob/miniredis v2.5.0+incompatible
-	github.com/apache/pulsar-client-go v0.8.1-0.20220429133321-5ee63303d43e
+	github.com/apache/pulsar-client-go v0.11.0
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
@@ -103,11 +103,11 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/DataDog/zstd v1.5.0 // indirect
 	github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
-	github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e // indirect
 	github.com/ardielle/ardielle-go v1.5.2 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aymanbagabas/go-osc52 v1.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.4.0 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/charmbracelet/lipgloss v0.6.1-0.20220911181249-6304a734e792 // indirect
@@ -169,8 +169,6 @@ require (
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.14.0 // indirect
-	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.19.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect