Merge branch 'master' into sendToGitHub/floating-resources

go.mod

@@ -38,7 +38,6 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
-	github.com/rakyll/statik v0.1.7
 	github.com/renstrom/shortuuid v3.0.0+incompatible
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0

go.sum

@@ -472,8 +472,6 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne
 github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
-github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5 h1:kvl0LOTQD23VR1R7A9vDti9msfV6mOE2+j6ngYkFsfg=
 github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5/go.mod h1:VhyLk7MdSTKbJCx6+wXlj3/ebh49gTq3yBiXymYrG7w=
 github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=

internal/armada/event/conversion/conversions.go

@@ -89,7 +89,7 @@ func FromInternalSubmit(owner string, groups []string, queue string, jobSet stri
 		JobSetId: jobSet,
 		Queue:    queue,
 		Created:  protoutil.ToTimestamp(time),
-		Job:      *job,
+		Job:      job,
 	}
 
 	queuedEvent := &api.JobQueuedEvent{

internal/armada/event/conversion/conversions_test.go

@@ -82,7 +82,7 @@ func TestConvertSubmitted(t *testing.T) {
 					JobSetId: jobSetName,
 					Queue:    queue,
 					Created:  protoutil.ToTimestamp(baseTime),
-					Job: api.Job{
+					Job: &api.Job{
 						Id:         jobIdString,
 						JobSetId:   jobSetName,
 						Queue:      queue,
@@ -99,7 +99,7 @@ func TestConvertSubmitted(t *testing.T) {
 								},
 							},
 						},
-						SchedulingResourceRequirements: v1.ResourceRequirements{
+						SchedulingResourceRequirements: &v1.ResourceRequirements{
 							Requests: make(v1.ResourceList),
 							Limits:   make(v1.ResourceList),
 						},
@@ -795,13 +795,13 @@ func TestConvertResourceUtilisation(t *testing.T) {
 						},
 					},
 				},
-				MaxResourcesForPeriod: map[string]resource.Quantity{
-					"cpu": resource.MustParse("2.0"),
-					"mem": resource.MustParse("100Gi"),
+				MaxResourcesForPeriod: map[string]*resource.Quantity{
+					"cpu": resourcePointer("2.0"),
+					"mem": resourcePointer("100Gi"),
 				},
-				TotalCumulativeUsage: map[string]resource.Quantity{
-					"cpu": resource.MustParse("3.0"),
-					"mem": resource.MustParse("200Gi"),
+				TotalCumulativeUsage: map[string]*resource.Quantity{
+					"cpu": resourcePointer("3.0"),
+					"mem": resourcePointer("200Gi"),
 				},
 			},
 		},
@@ -817,17 +817,17 @@ func TestConvertResourceUtilisation(t *testing.T) {
 					Created:      protoutil.ToTimestamp(baseTime),
 					ClusterId:    executorId,
 					KubernetesId: runIdString,
-					MaxResourcesForPeriod: map[string]resource.Quantity{
-						"cpu": resource.MustParse("2.0"),
-						"mem": resource.MustParse("100Gi"),
+					MaxResourcesForPeriod: map[string]*resource.Quantity{
+						"cpu": resourcePointer("2.0"),
+						"mem": resourcePointer("100Gi"),
 					},
 					NodeName:     nodeName,
 					PodNumber:    podNumber,
 					PodName:      podName,
 					PodNamespace: namespace,
-					TotalCumulativeUsage: map[string]resource.Quantity{
-						"cpu": resource.MustParse("3.0"),
-						"mem": resource.MustParse("200Gi"),
+					TotalCumulativeUsage: map[string]*resource.Quantity{
+						"cpu": resourcePointer("3.0"),
+						"mem": resourcePointer("200Gi"),
 					},
 				},
 			},
@@ -982,3 +982,8 @@ func toEventSeq(event ...*armadaevents.EventSequence_Event) *armadaevents.EventS
 		UserId:     userId,
 	}
 }
+
+func resourcePointer(s string) *resource.Quantity {
+	r := resource.MustParse(s)
+	return &r
+}

internal/armada/queue/queue_repository_test.go

@@ -10,19 +10,22 @@ import (
 
 	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/database/lookout"
+	"github.com/armadaproject/armada/pkg/api"
 	"github.com/armadaproject/armada/pkg/client/queue"
 )
 
 var (
 	queueA = queue.Queue{
-		Name:           "queueA",
-		PriorityFactor: 1000,
-		Permissions:    []queue.Permissions{},
+		Name:                              "queueA",
+		PriorityFactor:                    1000,
+		Permissions:                       []queue.Permissions{},
+		ResourceLimitsByPriorityClassName: map[string]api.PriorityClassResourceLimits{},
 	}
 	queueB = queue.Queue{
-		Name:           "queueB",
-		PriorityFactor: 2000,
-		Permissions:    []queue.Permissions{},
+		Name:                              "queueB",
+		PriorityFactor:                    2000,
+		Permissions:                       []queue.Permissions{},
+		ResourceLimitsByPriorityClassName: map[string]api.PriorityClassResourceLimits{},
 	}
 	twoQueues = []queue.Queue{queueA, queueB}
 )
@@ -111,17 +114,19 @@ func TestGetAndUpdateQueue(t *testing.T) {
 		"Queue Doesn't Exist": {
 			intialQueues: twoQueues,
 			queueToUpdate: queue.Queue{
-				Name:           "queueC",
-				PriorityFactor: 1,
-				Permissions:    []queue.Permissions{},
+				Name:                              "queueC",
+				Permissions:                       []queue.Permissions{},
+				PriorityFactor:                    1,
+				ResourceLimitsByPriorityClassName: map[string]api.PriorityClassResourceLimits{},
 			},
 		},
 		"Queue Does Exist": {
 			intialQueues: twoQueues,
 			queueToUpdate: queue.Queue{
-				Name:           "queueA",
-				PriorityFactor: queueA.PriorityFactor + 100,
-				Permissions:    []queue.Permissions{},
+				Name:                              "queueA",
+				PriorityFactor:                    queueA.PriorityFactor + 100,
+				Permissions:                       []queue.Permissions{},
+				ResourceLimitsByPriorityClassName: map[string]api.PriorityClassResourceLimits{},
 			},
 		},
 	}

internal/common/database/migrations.go

@@ -1,15 +1,12 @@
 package database
 
 import (
-	"bytes"
 	"io/fs"
 	"path"
 	"sort"
 	"strconv"
 	"strings"
 
-	stakikfs "github.com/rakyll/statik/fs"
-
 	"github.com/armadaproject/armada/internal/common/armadacontext"
 )
 
@@ -120,46 +117,3 @@ func ReadMigrations(fsys fs.FS, basePath string) ([]Migration, error) {
 	}
 	return migrations, nil
 }
-
-// TODO: remove this when we've migrated over to iofs
-func ReadMigrationsFromStatik(namespace string) ([]Migration, error) {
-	vfs, err := stakikfs.NewWithNamespace(namespace)
-	if err != nil {
-		return nil, err
-	}
-
-	dir, err := vfs.Open("/")
-	if err != nil {
-		return nil, err
-	}
-
-	files, err := dir.Readdir(-1)
-	if err != nil {
-		return nil, err
-	}
-
-	sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() })
-
-	var migrations []Migration
-	for _, f := range files {
-		file, err := vfs.Open("/" + f.Name())
-		if err != nil {
-			return nil, err
-		}
-		buf := new(bytes.Buffer)
-		_, err = buf.ReadFrom(file)
-		if err != nil {
-			return nil, err
-		}
-		id, err := strconv.Atoi(strings.Split(f.Name(), "_")[0])
-		if err != nil {
-			return nil, err
-		}
-		migrations = append(migrations, Migration{
-			id:   id,
-			name: f.Name(),
-			sql:  buf.String(),
-		})
-	}
-	return migrations, nil
-}

internal/common/eventutil/eventutil.go

@@ -162,7 +162,7 @@ func ApiJobFromLogSubmitJob(ownerId string, groups []string, queueName string, j
 
 		PodSpec:                        podSpec,
 		PodSpecs:                       podSpecs,
-		SchedulingResourceRequirements: schedulingResourceRequirements,
+		SchedulingResourceRequirements: &schedulingResourceRequirements,
 
 		Created:                  protoutil.ToTimestamp(time),
 		Owner:                    ownerId,

internal/common/metrics/domain.go

@@ -10,6 +10,7 @@ import (
 type QueueMetricProvider interface {
 	GetQueuedJobMetrics(queueName string) []*QueueMetrics
 	GetRunningJobMetrics(queueName string) []*QueueMetrics
+	GetQueuePriorites() map[string]float64
 }
 
 type QueueMetrics struct {