Merge branch 'master' of https://github.com/armadaproject/armada into…

… refactor-create-node

client/python/pyproject.toml

@@ -1,10 +1,10 @@
 [project]
 name = "armada_client"
-version = "0.3.4"
+version = "0.3.5"
 description = "Armada gRPC API python client"
 readme = "README.md"
 requires-python = ">=3.7"
-dependencies = ["grpcio==1.58.0", "grpcio-tools==1.58.0", "mypy-protobuf>=3.2.0", "protobuf>=3.20,<5.0" ]
+dependencies = ["grpcio==1.66.1", "grpcio-tools==1.66.1", "mypy-protobuf>=3.2.0", "protobuf>=5.26.1,<6.0dev" ]
 license = { text = "Apache Software License" }
 authors = [{ name = "G-Research Open Source Software", email = "armada@armadaproject.io" }]
 

cmd/armadactl/cmd/submit.go

@@ -17,11 +17,11 @@ func submitCmd() *cobra.Command {
 
 Example jobs.yaml:
 
+queue: test
+jobSetId: set1
 jobs:
-- queue: test
-	priority: 0
-	jobSetId: set1
-	podSpec:
+  - namespace: armada-example
+    podSpec:
 	... kubernetes pod spec ...`,
 		Args: cobra.ExactArgs(1),
 		PreRunE: func(cmd *cobra.Command, args []string) error {

cmd/scheduler/cmd/root.go

@@ -42,7 +42,7 @@ func loadConfig() (schedulerconfig.Configuration, error) {
 	common.LoadConfig(&config, "./config/scheduler", viper.GetStringSlice(CustomConfigLocation))
 	err := config.Validate()
 	if err != nil {
-		commonconfig.LogValidationErrors(err)
+		return config, commonconfig.FormatValidationErrors(err)
 	}
 	return config, err
 }

config/scheduler/config.yaml

@@ -111,6 +111,5 @@ scheduling:
       resolution: "1Gi"
   executorTimeout: "10m"
   maxUnacknowledgedJobsPerExecutor: 2500
-  alwaysAttemptScheduling: false
   executorUpdateFrequency: "60s"
 

go.mod

@@ -84,7 +84,6 @@ require (
 	github.com/segmentio/fasthash v1.0.3
 	github.com/xitongsys/parquet-go v1.6.2
 	golang.org/x/time v0.5.0
-	gonum.org/v1/gonum v0.14.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f
 	gopkg.in/inf.v0 v0.9.1
 )

go.sum

@@ -735,8 +735,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
-gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

internal/binoculars/configuration/validation.go

@@ -0,0 +1,8 @@
+package configuration
+
+import "github.com/go-playground/validator/v10"
+
+func (c BinocularsConfig) Validate() error {
+	validate := validator.New()
+	return validate.Struct(c)
+}

internal/common/config/pulsar.go

@@ -34,7 +34,7 @@ type PulsarConfig struct {
 	// Maximum allowed message size in bytes
 	MaxAllowedMessageSize uint
 	// Timeout when sending messages asynchronously
-	SendTimeout time.Duration `validate:"required"`
+	SendTimeout time.Duration
 	// Backoff from polling when Pulsar returns an error
 	BackoffTime time.Duration
 	// Number of pulsar messages that will be queued by the pulsar consumer.

internal/common/config/validation.go

@@ -1,25 +1,30 @@
 package config
 
 import (
+	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/go-playground/validator/v10"
-	log "github.com/sirupsen/logrus"
 )
 
-func LogValidationErrors(err error) {
-	if err != nil {
-		for _, err := range err.(validator.ValidationErrors) {
-			fieldName := stripPrefix(err.Namespace())
-			tag := err.Tag()
-			switch tag {
-			case "required":
-				log.Errorf("ConfigError: Field %s is required but was not found", fieldName)
-			default:
-				log.Errorf("ConfigError: Field %s has invalid value %s: %s", fieldName, err.Value(), tag)
-			}
+type Config interface {
+	Validate() error
+}
+
+func FormatValidationErrors(err error) error {
+	var validationErrors error
+	for _, err := range err.(validator.ValidationErrors) {
+		fieldName := stripPrefix(err.Namespace())
+		tag := err.Tag()
+		switch tag {
+		case "required":
+			validationErrors = errors.Join(validationErrors, fmt.Errorf("ConfigError: Field %s is required but was not found", fieldName))
+		default:
+			validationErrors = errors.Join(validationErrors, fmt.Errorf("ConfigError: Field %s has invalid value %s: %s", fieldName, err.Value(), tag))
 		}
 	}
+	return validationErrors
 }
 
 func stripPrefix(s string) string {

internal/common/grpc/configuration/types.go

@@ -5,7 +5,7 @@ import (
 )
 
 type GrpcConfig struct {
-	Port                       int `validate:"required"`
+	Port                       int
 	KeepaliveParams            keepalive.ServerParameters
 	KeepaliveEnforcementPolicy keepalive.EnforcementPolicy
 	Tls                        TlsConfig