diff --git a/config/clusters/nasa-veda/binder.values.yaml b/config/clusters/nasa-veda/binder.values.yaml index d39d68f53..61f9687b7 100644 --- a/config/clusters/nasa-veda/binder.values.yaml +++ b/config/clusters/nasa-veda/binder.values.yaml @@ -1,3 +1,6 @@ +userServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-binder jupyterhub: ingress: hosts: @@ -28,9 +31,7 @@ jupyterhub: extraVolumeMounts: [] singleuser: nodeSelector: - # Schedule users on the smallest instance - # https://github.com/2i2c-org/infrastructure/issues/4241 - node.kubernetes.io/instance-type: r5.xlarge + 2i2c/hub-name: "binder" memory: guarantee: 1G limit: 2G @@ -92,9 +93,7 @@ binderhub-service: enabled: true dockerApi: nodeSelector: - # Schedule dockerApi pods to run on the smallest user nodes only - # https://github.com/2i2c-org/infrastructure/issues/4241 - node.kubernetes.io/instance-type: r5.xlarge + 2i2c/hub-name: "binder" ingress: enabled: true hosts: [binder.openveda.cloud] @@ -109,6 +108,7 @@ binderhub-service: # Schedule builder pods to run on the smallest user nodes only # https://github.com/2i2c-org/infrastructure/issues/4241 node.kubernetes.io/instance-type: r5.xlarge + 2i2c/hub-name: "binder" BinderHub: base_url: / hub_url: https://hub.binder.nasa-veda.2i2c.cloud diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml index 24d6bc29f..ebf0c5502 100644 --- a/config/clusters/nasa-veda/prod.values.yaml +++ b/config/clusters/nasa-veda/prod.values.yaml @@ -12,7 +12,22 @@ basehub: homepage: gitRepoBranch: "master" gitRepoUrl: "https://github.com/NASA-IMPACT/veda-hub-homepage" + singleuser: + nodeSelector: + 2i2c/hub-name: prod hub: config: GitHubOAuthenticator: oauth_callback_url: https://hub.openveda.cloud/hub/oauth_callback + +dask-gateway: + gateway: + backend: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: prod + worker: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: prod diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml index 1ec06f776..18c9622b5 100644 --- a/config/clusters/nasa-veda/staging.values.yaml +++ b/config/clusters/nasa-veda/staging.values.yaml @@ -4,6 +4,8 @@ basehub: eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-staging jupyterhub: singleuser: + nodeSelector: + 2i2c/hub-name: staging initContainers: - &volume_ownership_fix_initcontainer name: volume-mount-ownership-fix @@ -39,3 +41,15 @@ basehub: homepage: gitRepoBranch: "staging" gitRepoUrl: "https://github.com/NASA-IMPACT/veda-hub-homepage" + +dask-gateway: + gateway: + backend: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: staging + worker: + extraPodConfig: + node_selector: + 2i2c/hub-name: staging diff --git a/eksctl/nasa-veda.jsonnet b/eksctl/nasa-veda.jsonnet index 70b29cf05..a62b37989 100644 --- a/eksctl/nasa-veda.jsonnet +++ b/eksctl/nasa-veda.jsonnet @@ -25,9 +25,50 @@ local nodeAz = "us-west-2a"; // A `node.kubernetes.io/instance-type label is added, so pods // can request a particular kind of node with a nodeSelector local notebookNodes = [ - { instanceType: "r5.xlarge" }, - { instanceType: "r5.4xlarge" }, - { instanceType: "r5.16xlarge" }, + { instanceType: "r5.xlarge" }, // FIXME: tainted, to be deleted when empty, replaced by equivalent + { instanceType: "r5.xlarge", nameSuffix: "b" }, // FIXME: tainted, to be deleted when empty, replaced by equivalent + { + instanceType: "r5.xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" } + }, + { + instanceType: "r5.4xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" } + }, + { + instanceType: "r5.16xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" } + }, + { + instanceType: "r5.xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" } + }, + { + instanceType: "r5.4xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" } + }, + { + instanceType: "r5.16xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" } + }, + { + instanceType: "r5.xlarge", + namePrefix: "nb-binder", + labels+: { "2i2c/hub-name": "binder" }, + tags+: { "2i2c:hub-name": "binder" } + } ]; local daskNodes = [ @@ -41,7 +82,18 @@ local daskNodes = [ // A not yet fully established policy is being developed about using a single // node pool, see https://github.com/2i2c-org/infrastructure/issues/2687. // - { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }}, + { + namePrefix: "dask-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" }, + instancesDistribution+: { instanceTypes: ["r5.4xlarge"] } + }, + { + namePrefix: "dask-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" }, + instancesDistribution+: { instanceTypes: ["r5.4xlarge"] } + }, ]; @@ -80,7 +132,7 @@ local daskNodes = [ nodeGroups: [ ng + { namePrefix: 'core', - nameSuffix: 'b', + nameSuffix: 'a', nameIncludeInstanceType: false, availabilityZones: [nodeAz], ssh: { @@ -93,6 +145,9 @@ local daskNodes = [ "hub.jupyter.org/node-purpose": "core", "k8s.dask.org/node-purpose": "core" }, + tags+: { + "2i2c:node-purpose": "core" + }, }, ] + [ ng + { @@ -108,6 +163,9 @@ local daskNodes = [ "hub.jupyter.org/node-purpose": "user", "k8s.dask.org/node-purpose": "scheduler" }, + tags+: { + "2i2c:node-purpose": "user" + }, taints+: { "hub.jupyter.org_dedicated": "user:NoSchedule", "hub.jupyter.org/dedicated": "user:NoSchedule" @@ -126,6 +184,9 @@ local daskNodes = [ labels+: { "k8s.dask.org/node-purpose": "worker" }, + tags+: { + "2i2c:node-purpose": "worker" + }, taints+: { "k8s.dask.org_dedicated" : "worker:NoSchedule", "k8s.dask.org/dedicated" : "worker:NoSchedule" diff --git a/terraform/aws/projects/nasa-veda.tfvars b/terraform/aws/projects/nasa-veda.tfvars index 3d64aba58..de2decdd2 100644 --- a/terraform/aws/projects/nasa-veda.tfvars +++ b/terraform/aws/projects/nasa-veda.tfvars @@ -18,6 +18,9 @@ user_buckets = { "scratch" : { "delete_after" : 7 }, + "scratch-binder" : { + "delete_after" : 1 + }, } @@ -150,4 +153,70 @@ hub_cloud_permissions = { EOT }, }, + "binder" : { + "user-sa" : { + bucket_admin_access : ["scratch-binder"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:CreateBucket", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::veda-nex-gddp-cmip6-public", + "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", + "arn:aws:s3:::cmip6-staging", + "arn:aws:s3:::cmip6-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::pangeo-forge-veda-output", + "arn:aws:s3:::pangeo-forge-veda-output/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*", + "arn:aws:s3:::maap-ops-workspace", + "arn:aws:s3:::maap-ops-workspace/*", + "arn:aws:s3:::nasa-maap-data-store", + "arn:aws:s3:::nasa-maap-data-store/*", + "arn:aws:s3:::sdap-dev-zarr", + "arn:aws:s3:::sdap-dev-zarr/*", + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*", + "arn:aws:s3:::sentinel-cogs", + "arn:aws:s3:::sentinel-cogs/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT + }, + }, }