Merge branch 'master' of https://github.com/armadaproject/armada into…

… severinson/unified-scheduling

config/armada/config.yaml

@@ -9,6 +9,8 @@ cancelJobsBatchSize: 1000
 pulsarSchedulerEnabled: false
 probabilityOfUsingPulsarScheduler: 0
 ignoreJobSubmitChecks: false
+schedulerApiConnection:
+  armadaUrl: "localhost:50052"
 grpc:
   keepaliveParams:
     maxConnectionIdle: 5m
@@ -17,6 +19,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 10s
     permitWithoutStream: true
+  tls:
+    enabled: false
 redis:
   addrs:
     - redis:6379

config/binoculars/config.yaml

@@ -1,7 +1,7 @@
 grpcPort: 50051
 httpPort: 8080
 metricsPort: 9000
-corsAllowedOrigins: 
+corsAllowedOrigins:
   - http://localhost:3000
   - http://localhost:8080
 cordon:
@@ -24,3 +24,5 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 5m
     permitWithoutStream: false
+  tls:
+    enabled: false

config/jobservice/config.yaml

@@ -28,6 +28,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 5m
     permitWithoutStream: false
+  tls:
+    enabled: false
 # gRPC connection pool to armada server configuration.
 grpcPool:
   initialConnections: 5

config/scheduler/config.yaml

@@ -37,6 +37,8 @@ leader:
   renewDeadline: 10s
   retryPeriod: 2s
   podName: "" # This must be set so viper allows env vars to overwrite it
+  leaderConnection:
+    armadaUrl: "" # <name> will get replaced with the lease owners name
 http:
   port: 8080
 grpc:
@@ -48,6 +50,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 10s
     permitWithoutStream: true
+  tls:
+    enabled: false
 scheduling:
   executorTimeout: 10m
   enableAssertions: true

deployment/armada/templates/deployment.yaml

@@ -76,6 +76,11 @@ spec:
               mountPath: "/pulsar/ca"
               readOnly: true
             {{- end }}
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -129,6 +134,11 @@ spec:
               - key: ca.crt
                 path: ca.crt
         {{- end }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: armada-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}

deployment/armada/templates/ingress.yaml

@@ -7,7 +7,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

deployment/armada/templates/ingressrest.yaml

@@ -7,6 +7,10 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     nginx.ingress.kubernetes.io/rewrite-target: /$2

deployment/armada/values.yaml

@@ -84,6 +84,11 @@ serviceAccount: {}
 applicationConfig:
   # -- Armada Server gRPC port
   grpcPort: 50051
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key
   # -- Armada Server REST port
   httpPort: 8080
   pulsar:

deployment/binoculars/templates/deployment.yaml

@@ -59,6 +59,11 @@ spec:
               mountPath: /config/application_config.yaml
               subPath: {{ include "binoculars.config.filename" . }}
               readOnly: true
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -94,6 +99,11 @@ spec:
         - name: user-config
           secret:
             secretName: {{ include "binoculars.config.name" . }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: binoculars-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}

deployment/binoculars/templates/ingress.yaml

@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

deployment/binoculars/templates/ingressrest.yaml

@@ -6,6 +6,10 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     nginx.ingress.kubernetes.io/rewrite-target: /$2

deployment/binoculars/values.yaml

@@ -9,7 +9,7 @@ resources:
     memory: 512Mi
     cpu: 200m
 # -- Tolerations
-tolerations: []    
+tolerations: []
 additionalLabels: {}
 additionalClusterRoleBindings: []
 additionalVolumeMounts: []
@@ -32,5 +32,10 @@ serviceAccount: null
 
 applicationConfig:
   grpcPort: 50051
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key
   httpPort: 8080
   metricsPort: 9000

deployment/jobservice/templates/ingress.yaml

@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

deployment/jobservice/templates/statefulset.yaml

@@ -49,6 +49,11 @@ spec:
               mountPath: /config/application_config.yaml
               subPath: {{ include "jobservice.config.filename" . }}
               readOnly: true
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -71,6 +76,11 @@ spec:
         - name: user-config
           secret:
             secretName: {{ include "jobservice.config.name" . }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: jobservice-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}

deployment/jobservice/values.yaml

@@ -9,7 +9,7 @@ resources:
     memory: 512Mi
     cpu: 200m
 # -- Tolerations
-tolerations: []    
+tolerations: []
 additionalLabels: {}
 terminationGracePeriodSeconds: 30
 replicas: 1
@@ -30,3 +30,8 @@ serviceAccount: null
 
 applicationConfig:
   grpcPort: 60063
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key

deployment/scheduler/templates/scheduler-clusterrolebinding.yaml

@@ -14,7 +14,7 @@ subjects:
     name: {{ .Values.scheduler.customServiceAccount | default (include "armada-scheduler.name" .) }}
     namespace: {{ .Release.Namespace }}
 {{- $root := . -}}
-{{ range .Values.additionalClusterRoleBindings }}
+{{ range .Values.scheduler.additionalClusterRoleBindings }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

deployment/scheduler/templates/scheduler-ingress.yaml

@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.scheduler.ingressClass" .Values.scheduler.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.scheduler.clusterIssuer" .Values.scheduler.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.scheduler.clusterIssuer" .Values.scheduler.clusterIssuer }}
     {{- if .Values.scheduler.ingress.annotations }}

deployment/scheduler/templates/scheduler-statefulset.yaml

@@ -49,13 +49,17 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            {{- if  not (((.Values.scheduler.applicationConfig).leader).leaderConnection).armadaUrl }}
+            - name: ARMADA_LEADER_LEADERCONNECTION_ARMADAURL
+              value: "<name>.{{ include "armada-scheduler.name" . }}.{{ .Release.Namespace }}.svc:{{ .Values.scheduler.applicationConfig.grpc.port }}"
+            {{- end }}
           {{- if .Values.scheduler.env }}
             {{- toYaml .Values.scheduler.env | nindent 12 -}}
           {{- end }}
           resources:
             {{- toYaml .Values.scheduler.resources | nindent 12 }}
           ports:
-            - containerPort: { { .Values.scheduler.applicationConfig.http.port } }
+            - containerPort: {{ .Values.scheduler.applicationConfig.http.port }}
               protocol: TCP
               name: http
             - containerPort:  {{ .Values.scheduler.applicationConfig.grpc.port }}
@@ -79,6 +83,11 @@ spec:
               mountPath: "/pulsar/ca"
               readOnly: true
             {{- end }}
+            {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.scheduler.additionalVolumeMounts }}
             {{- toYaml .Values.scheduler.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -104,6 +113,11 @@ spec:
                         - {{ include "armada-scheduler.name" . }}
                 topologyKey: kubernetes.io/hostname
       volumes:
+        {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: armada-scheduler-service-tls
+        {{- end}}
         - name: user-config
           secret:
             secretName: {{ include "armada-scheduler.config.name" . }}

deployment/scheduler/values.yaml

@@ -17,6 +17,10 @@ scheduler:
   applicationConfig:
     grpc:
       port: 50051
+      tls:
+        enabled: false
+        certPath: /certs/tls.crt
+        keyPath: /certs/tls.key
     metrics:
       port: 9001
     http:

internal/armada/configuration/types.go

@@ -12,6 +12,8 @@ import (
 	authconfig "github.com/armadaproject/armada/internal/common/auth/configuration"
 	grpcconfig "github.com/armadaproject/armada/internal/common/grpc/configuration"
 	armadaresource "github.com/armadaproject/armada/internal/common/resource"
+	"github.com/armadaproject/armada/internal/common/types"
+	"github.com/armadaproject/armada/pkg/client"
 )
 
 type ArmadaConfig struct {
@@ -27,6 +29,8 @@ type ArmadaConfig struct {
 
 	Grpc grpcconfig.GrpcConfig
 
+	SchedulerApiConnection client.ApiConnectionDetails
+
 	PriorityHalfTime                  time.Duration
 	CancelJobsBatchSize               int
 	Redis                             redis.UniversalOptions
@@ -246,31 +250,19 @@ type PreemptionConfig struct {
 	// Map from priority class names to priority classes.
 	// Must be consistent with Kubernetes priority classes.
 	// I.e., priority classes defined here must be defined in all executor clusters and should map to the same priority.
-	PriorityClasses map[string]PriorityClass
+	PriorityClasses map[string]types.PriorityClass
 	// Priority class assigned to pods that do not specify one.
 	// Must be an entry in PriorityClasses above.
 	DefaultPriorityClass string
 	// If set, override the priority class name of pods with this value when sending to an executor.
 	PriorityClassNameOverride *string
 }
 
-type PriorityClass struct {
-	Priority int32
-	// If true, Armada may preempt jobs of this class to improve fairness.
-	Preemptible bool
-	// Limits resources assigned to jobs of this priority class.
-	// Specifically, jobs of this priority class are only scheduled if doing so does not exceed this limit.
-	MaximumResourceFractionPerQueue map[string]float64
-	// Per-pool override of MaximumResourceFractionPerQueue.
-	// If missing for a particular pool, MaximumResourceFractionPerQueue is used instead for that pool.
-	MaximumResourceFractionPerQueueByPool map[string]map[string]float64
-}
-
 func (p PreemptionConfig) PriorityByPriorityClassName() map[string]int32 {
 	return PriorityByPriorityClassName(p.PriorityClasses)
 }
 
-func PriorityByPriorityClassName(priorityClasses map[string]PriorityClass) map[string]int32 {
+func PriorityByPriorityClassName(priorityClasses map[string]types.PriorityClass) map[string]int32 {
 	rv := make(map[string]int32, len(priorityClasses))
 	for name, pc := range priorityClasses {
 		rv[name] = pc.Priority
@@ -282,7 +274,7 @@ func (p PreemptionConfig) AllowedPriorities() []int32 {
 	return AllowedPriorities(p.PriorityClasses)
 }
 
-func AllowedPriorities(priorityClasses map[string]PriorityClass) []int32 {
+func AllowedPriorities(priorityClasses map[string]types.PriorityClass) []int32 {
 	rv := make([]int32, 0, len(priorityClasses))
 	for _, v := range priorityClasses {
 		rv = append(rv, v.Priority)