Merge pull request #4595 from GeorgianaElena/binder.nasa-veda.2i2c.cloud

[veda-binder] Update the underlying infra
2i2c-org · Aug 9, 2024 · 726a441 · 726a441
2 parents 0c18a1e + 2e5cbaa
commit 726a441
Show file tree

Hide file tree

Showing 5 changed files with 170 additions and 11 deletions.
diff --git a/config/clusters/nasa-veda/binder.values.yaml b/config/clusters/nasa-veda/binder.values.yaml
@@ -1,3 +1,6 @@
+userServiceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-binder
 jupyterhub:
   ingress:
     hosts:
@@ -28,9 +31,7 @@ jupyterhub:
       extraVolumeMounts: []
   singleuser:
     nodeSelector:
-      # Schedule users on the smallest instance
-      # https://github.com/2i2c-org/infrastructure/issues/4241
-      node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: "binder"
     memory:
       guarantee: 1G
       limit: 2G
@@ -92,9 +93,7 @@ binderhub-service:
   enabled: true
   dockerApi:
     nodeSelector:
-      # Schedule dockerApi pods to run on the smallest user nodes only
-      # https://github.com/2i2c-org/infrastructure/issues/4241
-      node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: "binder"
   ingress:
     enabled: true
     hosts: [binder.openveda.cloud]
@@ -109,6 +108,7 @@ binderhub-service:
         # Schedule builder pods to run on the smallest user nodes only
         # https://github.com/2i2c-org/infrastructure/issues/4241
         node.kubernetes.io/instance-type: r5.xlarge
+        2i2c/hub-name: "binder"
     BinderHub:
       base_url: /
       hub_url: https://hub.binder.nasa-veda.2i2c.cloud

diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml
@@ -12,7 +12,22 @@ basehub:
       homepage:
         gitRepoBranch: "master"
         gitRepoUrl: "https://github.com/NASA-IMPACT/veda-hub-homepage"
+    singleuser:
+      nodeSelector:
+        2i2c/hub-name: prod
     hub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://hub.openveda.cloud/hub/oauth_callback
+
+dask-gateway:
+  gateway:
+    backend:
+      scheduler:
+        extraPodConfig:
+          nodeSelector:
+            2i2c/hub-name: prod
+      worker:
+        extraPodConfig:
+          nodeSelector:
+            2i2c/hub-name: prod
diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml
@@ -4,6 +4,8 @@ basehub:
       eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-staging
   jupyterhub:
     singleuser:
+      nodeSelector:
+        2i2c/hub-name: staging
       initContainers:
         - &volume_ownership_fix_initcontainer
           name: volume-mount-ownership-fix
@@ -39,3 +41,15 @@ basehub:
       homepage:
         gitRepoBranch: "staging"
         gitRepoUrl: "https://github.com/NASA-IMPACT/veda-hub-homepage"
+
+dask-gateway:
+  gateway:
+    backend:
+      scheduler:
+        extraPodConfig:
+          nodeSelector:
+            2i2c/hub-name: staging
+      worker:
+        extraPodConfig:
+          node_selector:
+            2i2c/hub-name: staging
diff --git a/eksctl/nasa-veda.jsonnet b/eksctl/nasa-veda.jsonnet
@@ -25,9 +25,50 @@ local nodeAz = "us-west-2a";
 // A `node.kubernetes.io/instance-type label is added, so pods
 // can request a particular kind of node with a nodeSelector
 local notebookNodes = [
-    { instanceType: "r5.xlarge" },
-    { instanceType: "r5.4xlarge" },
-    { instanceType: "r5.16xlarge" },
+    { instanceType: "r5.xlarge" },  // FIXME: tainted, to be deleted when empty, replaced by equivalent
+    { instanceType: "r5.xlarge", nameSuffix: "b" }, // FIXME: tainted, to be deleted when empty, replaced by equivalent
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" }
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" }
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" }
+    },
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" }
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" }
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" }
+    },
+    {
+      instanceType: "r5.xlarge",
+      namePrefix: "nb-binder",
+      labels+: { "2i2c/hub-name": "binder" },
+      tags+: { "2i2c:hub-name": "binder" }
+    }
 ];
 
 local daskNodes = [
@@ -41,7 +82,18 @@ local daskNodes = [
     // A not yet fully established policy is being developed about using a single
     // node pool, see https://github.com/2i2c-org/infrastructure/issues/2687.
     //
-    { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }},
+    {  
+        namePrefix: "dask-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+        instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }
+    },
+    {  
+        namePrefix: "dask-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+        instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }
+    },
 ];
 
 
@@ -80,7 +132,7 @@ local daskNodes = [
     nodeGroups: [
         ng + {
             namePrefix: 'core',
-            nameSuffix: 'b',
+            nameSuffix: 'a',
             nameIncludeInstanceType: false,
             availabilityZones: [nodeAz],
             ssh: {
@@ -93,6 +145,9 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "core",
                 "k8s.dask.org/node-purpose": "core"
             },
+            tags+: {
+                "2i2c:node-purpose": "core"
+            },
         },
     ] + [
         ng + {
@@ -108,6 +163,9 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "user",
                 "k8s.dask.org/node-purpose": "scheduler"
             },
+            tags+: {
+                "2i2c:node-purpose": "user"
+            },
             taints+: {
                 "hub.jupyter.org_dedicated": "user:NoSchedule",
                 "hub.jupyter.org/dedicated": "user:NoSchedule"
@@ -126,6 +184,9 @@ local daskNodes = [
             labels+: {
                 "k8s.dask.org/node-purpose": "worker"
             },
+            tags+: {
+                "2i2c:node-purpose": "worker"
+            },
             taints+: {
                 "k8s.dask.org_dedicated" : "worker:NoSchedule",
                 "k8s.dask.org/dedicated" : "worker:NoSchedule"

diff --git a/terraform/aws/projects/nasa-veda.tfvars b/terraform/aws/projects/nasa-veda.tfvars
@@ -18,6 +18,9 @@ user_buckets = {
   "scratch" : {
     "delete_after" : 7
   },
+  "scratch-binder" : {
+    "delete_after" : 1
+  },
 }
 
 
@@ -150,4 +153,70 @@ hub_cloud_permissions = {
       EOT
     },
   },
+  "binder" : {
+    "user-sa" : {
+      bucket_admin_access : ["scratch-binder"],
+      extra_iam_policy : <<-EOT
+        {
+          "Version": "2012-10-17",
+          "Statement": [
+            {
+              "Effect": "Allow",
+              "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:ListBucketMultipartUploads",
+                "s3:AbortMultipartUpload",
+                "s3:ListBucketVersions",
+                "s3:CreateBucket",
+                "s3:ListBucket",
+                "s3:DeleteObject",
+                "s3:GetBucketLocation",
+                "s3:ListMultipartUploadParts"
+              ],
+              "Resource": [
+                "arn:aws:s3:::veda-data-store",
+                "arn:aws:s3:::veda-data-store/*",
+                "arn:aws:s3:::veda-data-store-staging",
+                "arn:aws:s3:::veda-data-store-staging/*",
+                "arn:aws:s3:::veda-nex-gddp-cmip6-public",
+                "arn:aws:s3:::veda-nex-gddp-cmip6-public/*",
+                "arn:aws:s3:::cmip6-staging",
+                "arn:aws:s3:::cmip6-staging/*",
+                "arn:aws:s3:::lp-prod-protected",
+                "arn:aws:s3:::lp-prod-protected/*",
+                "arn:aws:s3:::gesdisc-cumulus-prod-protected",
+                "arn:aws:s3:::gesdisc-cumulus-prod-protected/*",
+                "arn:aws:s3:::nsidc-cumulus-prod-protected",
+                "arn:aws:s3:::nsidc-cumulus-prod-protected/*",
+                "arn:aws:s3:::ornl-cumulus-prod-protected",
+                "arn:aws:s3:::ornl-cumulus-prod-protected/*",
+                "arn:aws:s3:::pangeo-forge-veda-output",
+                "arn:aws:s3:::pangeo-forge-veda-output/*",
+                "arn:aws:s3:::podaac-ops-cumulus-public",
+                "arn:aws:s3:::podaac-ops-cumulus-public/*",
+                "arn:aws:s3:::podaac-ops-cumulus-protected",
+                "arn:aws:s3:::podaac-ops-cumulus-protected/*",
+                "arn:aws:s3:::maap-ops-workspace",
+                "arn:aws:s3:::maap-ops-workspace/*",
+                "arn:aws:s3:::nasa-maap-data-store",
+                "arn:aws:s3:::nasa-maap-data-store/*",
+                "arn:aws:s3:::sdap-dev-zarr",
+                "arn:aws:s3:::sdap-dev-zarr/*",
+                "arn:aws:s3:::usgs-landsat",
+                "arn:aws:s3:::usgs-landsat/*",
+                "arn:aws:s3:::sentinel-cogs",
+                "arn:aws:s3:::sentinel-cogs/*"
+              ]
+            },
+            {
+              "Effect": "Allow",
+              "Action": "s3:ListAllMyBuckets",
+              "Resource": "*"
+            }
+          ]
+        }
+      EOT
+    },
+  },
 }