Merge branch 'master' into armada-scheduler-helm-fixes

armadaproject · Jul 11, 2023 · c6454ba · c6454ba
2 parents f1314c2 + 1e2e721
commit c6454ba
Show file tree

Hide file tree

Showing 27 changed files with 455 additions and 13 deletions.
diff --git a/config/armada/config.yaml b/config/armada/config.yaml
@@ -19,6 +19,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 10s
     permitWithoutStream: true
+  tls:
+    enabled: false
 redis:
   addrs:
     - redis:6379

diff --git a/config/binoculars/config.yaml b/config/binoculars/config.yaml
@@ -1,7 +1,7 @@
 grpcPort: 50051
 httpPort: 8080
 metricsPort: 9000
-corsAllowedOrigins: 
+corsAllowedOrigins:
   - http://localhost:3000
   - http://localhost:8080
 cordon:
@@ -24,3 +24,5 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 5m
     permitWithoutStream: false
+  tls:
+    enabled: false
diff --git a/config/jobservice/config.yaml b/config/jobservice/config.yaml
@@ -28,6 +28,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 5m
     permitWithoutStream: false
+  tls:
+    enabled: false
 # gRPC connection pool to armada server configuration.
 grpcPool:
   initialConnections: 5

diff --git a/config/scheduler/config.yaml b/config/scheduler/config.yaml
@@ -50,6 +50,8 @@ grpc:
   keepaliveEnforcementPolicy:
     minTime: 10s
     permitWithoutStream: true
+  tls:
+    enabled: false
 scheduling:
   executorTimeout: 10m
   enableAssertions: true

diff --git a/deployment/armada/templates/deployment.yaml b/deployment/armada/templates/deployment.yaml
@@ -76,6 +76,11 @@ spec:
               mountPath: "/pulsar/ca"
               readOnly: true
             {{- end }}
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -129,6 +134,11 @@ spec:
               - key: ca.crt
                 path: ca.crt
         {{- end }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: armada-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}
diff --git a/deployment/armada/templates/ingress.yaml b/deployment/armada/templates/ingress.yaml
@@ -7,7 +7,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

diff --git a/deployment/armada/templates/ingressrest.yaml b/deployment/armada/templates/ingressrest.yaml
@@ -7,6 +7,10 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     nginx.ingress.kubernetes.io/rewrite-target: /$2

diff --git a/deployment/armada/values.yaml b/deployment/armada/values.yaml
@@ -84,6 +84,11 @@ serviceAccount: {}
 applicationConfig:
   # -- Armada Server gRPC port
   grpcPort: 50051
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key
   # -- Armada Server REST port
   httpPort: 8080
   pulsar:

diff --git a/deployment/binoculars/templates/deployment.yaml b/deployment/binoculars/templates/deployment.yaml
@@ -59,6 +59,11 @@ spec:
               mountPath: /config/application_config.yaml
               subPath: {{ include "binoculars.config.filename" . }}
               readOnly: true
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -94,6 +99,11 @@ spec:
         - name: user-config
           secret:
             secretName: {{ include "binoculars.config.name" . }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: binoculars-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}
diff --git a/deployment/binoculars/templates/ingress.yaml b/deployment/binoculars/templates/ingress.yaml
@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

diff --git a/deployment/binoculars/templates/ingressrest.yaml b/deployment/binoculars/templates/ingressrest.yaml
@@ -6,6 +6,10 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     nginx.ingress.kubernetes.io/rewrite-target: /$2

diff --git a/deployment/binoculars/values.yaml b/deployment/binoculars/values.yaml
@@ -9,7 +9,7 @@ resources:
     memory: 512Mi
     cpu: 200m
 # -- Tolerations
-tolerations: []    
+tolerations: []
 additionalLabels: {}
 additionalClusterRoleBindings: []
 additionalVolumeMounts: []
@@ -32,5 +32,10 @@ serviceAccount: null
 
 applicationConfig:
   grpcPort: 50051
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key
   httpPort: 8080
   metricsPort: 9000
diff --git a/deployment/jobservice/templates/ingress.yaml b/deployment/jobservice/templates/ingress.yaml
@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.ingressClass" .Values.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.clusterIssuer" .Values.clusterIssuer }}
     {{- if .Values.ingress.annotations }}

diff --git a/deployment/jobservice/templates/statefulset.yaml b/deployment/jobservice/templates/statefulset.yaml
@@ -49,6 +49,11 @@ spec:
               mountPath: /config/application_config.yaml
               subPath: {{ include "jobservice.config.filename" . }}
               readOnly: true
+            {{- if .Values.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.additionalVolumeMounts }}
             {{- toYaml .Values.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -71,6 +76,11 @@ spec:
         - name: user-config
           secret:
             secretName: {{ include "jobservice.config.name" . }}
+        {{- if .Values.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: jobservice-service-tls
+        {{- end }}
         {{- if .Values.additionalVolumes }}
         {{- toYaml .Values.additionalVolumes | nindent 8 }}
         {{- end }}
diff --git a/deployment/jobservice/values.yaml b/deployment/jobservice/values.yaml
@@ -9,7 +9,7 @@ resources:
     memory: 512Mi
     cpu: 200m
 # -- Tolerations
-tolerations: []    
+tolerations: []
 additionalLabels: {}
 terminationGracePeriodSeconds: 30
 replicas: 1
@@ -30,3 +30,8 @@ serviceAccount: null
 
 applicationConfig:
   grpcPort: 60063
+  grpc:
+    tls:
+      enabled: false
+      certPath: /certs/tls.crt
+      keyPath: /certs/tls.key
diff --git a/deployment/scheduler/templates/scheduler-ingress.yaml b/deployment/scheduler/templates/scheduler-ingress.yaml
@@ -6,7 +6,12 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: {{ required "A value is required for .Values.scheduler.ingressClass" .Values.scheduler.ingressClass }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    {{- else }}
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
     certmanager.k8s.io/cluster-issuer: {{ required "A value is required for .Values.scheduler.clusterIssuer" .Values.scheduler.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ required "A value is required for .Values.scheduler.clusterIssuer" .Values.scheduler.clusterIssuer }}
     {{- if .Values.scheduler.ingress.annotations }}

diff --git a/deployment/scheduler/templates/scheduler-statefulset.yaml b/deployment/scheduler/templates/scheduler-statefulset.yaml
@@ -91,6 +91,11 @@ spec:
               mountPath: "/pulsar/ca"
               readOnly: true
             {{- end }}
+            {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+            - name: tls-certs
+              mountPath: /certs
+              readOnly: true
+            {{- end }}
             {{- if .Values.scheduler.additionalVolumeMounts }}
             {{- toYaml .Values.scheduler.additionalVolumeMounts | nindent 12 -}}
             {{- end }}
@@ -116,6 +121,11 @@ spec:
                         - {{ include "armada-scheduler.name" . }}
                 topologyKey: kubernetes.io/hostname
       volumes:
+        {{- if .Values.scheduler.applicationConfig.grpc.tls.enabled }}
+        - name: tls-certs
+          secret:
+            secretName: armada-scheduler-service-tls
+        {{- end}}
         - name: user-config
           secret:
             secretName: {{ include "armada-scheduler.config.name" . }}

diff --git a/deployment/scheduler/values.yaml b/deployment/scheduler/values.yaml
@@ -17,6 +17,10 @@ scheduler:
   applicationConfig:
     grpc:
       port: 50051
+      tls:
+        enabled: false
+        certPath: /certs/tls.crt
+        keyPath: /certs/tls.key
     metrics:
       port: 9001
     http:

diff --git a/internal/armada/server.go b/internal/armada/server.go
@@ -77,7 +77,7 @@ func Serve(ctx context.Context, config *configuration.ArmadaConfig, healthChecks
 	if err != nil {
 		return err
 	}
-	grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices)
+	grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices, config.Grpc.Tls)
 
 	// Shut down grpcServer if the context is cancelled.
 	// Give the server 5 seconds to shut down gracefully.

diff --git a/internal/binoculars/server.go b/internal/binoculars/server.go
@@ -37,7 +37,7 @@ func StartUp(config *configuration.BinocularsConfig) (func(), *sync.WaitGroup) {
 		os.Exit(-1)
 	}
 
-	grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices)
+	grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices, config.Grpc.Tls)
 
 	permissionsChecker := authorization.NewPrincipalPermissionChecker(
 		config.Auth.PermissionGroupMapping,