Merge pull request #4773 from consideRatio/pr/nmfs-openscapes-staging…

…-prod

nmfs-openscapes: add staging and prod hub

.github/workflows/deploy-hubs.yaml

@@ -245,6 +245,7 @@ jobs:
       failure_nasa-esdis: "${{ env.failure_nasa-esdis }}"
       failure_nasa-ghg: "${{ env.failure_nasa-ghg }}"
       failure_nasa-veda: "${{ env.failure_nasa-veda }}"
+      failure_nmfs-openscapes: "${{ env.failure_nmfs-openscapes }}"
       failure_openscapes: "${{ env.failure_openscapes }}"
       failure_opensci: "${{ env.failure_opensci }}"
       failure_pangeo-hubs: "${{ env.failure_pangeo-hubs }}"

config/clusters/nmfs-openscapes/cluster.yaml

@@ -12,16 +12,19 @@ support:
     - support.values.yaml
     - enc-support.secret.values.yaml
 hubs:
-  []
-  # Uncomment the lines below once the support infrastructure was deployed and
-  # you are ready to add the first cluster
-
-  # - name: <hub_name>
-  #   # Tip: consider changing this to something more human friendly
-  #   display_name: "nmfs-openscapes - <hub_name>"
-  #   domain: <hub_name>.nmfs-openscapes.2i2c.cloud
-  #   helm_chart: basehub
-  #   helm_chart_values_files:
-  #     - common.values.yaml
-  #     - <hub_name>.values.yaml
-  #     - enc-<hub_name>.secret.values.yaml
+  - name: staging
+    display_name: NOAA Fisheries Openscapes - Staging
+    domain: staging.nmfs-openscapes.2i2c.cloud
+    helm_chart: basehub
+    helm_chart_values_files:
+      - common.values.yaml
+      - staging.values.yaml
+      - enc-staging.secret.values.yaml
+  - name: prod
+    display_name: NOAA Fisheries Openscapes
+    domain: nmfs-openscapes.2i2c.cloud
+    helm_chart: basehub
+    helm_chart_values_files:
+      - common.values.yaml
+      - staging.values.yaml
+      - enc-staging.secret.values.yaml

config/clusters/nmfs-openscapes/common.values.yaml

@@ -0,0 +1,197 @@
+nfs:
+  enabled: true
+  pv:
+    enabled: true
+    # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html
+    mountOptions:
+      - rsize=1048576
+      - wsize=1048576
+      - timeo=600
+      - soft # We pick soft over hard, so NFS lockups don't lead to hung processes
+      - retrans=2
+      - noresvport
+    baseShareName: /
+    # serverIP is set in staging / prod respectively
+
+jupyterhub:
+  custom:
+    2i2c:
+      add_staff_user_ids_to_admin_users: true
+      add_staff_user_ids_of_type: github
+    jupyterhubConfigurator:
+      enabled: false
+    homepage:
+      templateVars:
+        org:
+          name: NOAA Fisheries Openscapes
+          logo_url: https://raw.githubusercontent.com/nmfs-openscapes/nmfs-openscapes.github.io/main/images/logo-transparent-crop.png
+          url: https://nmfs-openscapes.github.io/
+        designed_by:
+          name: 2i2c
+          url: https://2i2c.org
+        operated_by:
+          name: 2i2c
+          url: https://2i2c.org
+        funded_by:
+          name: NOAA Fisheries and the Biden-Harris Administration's Inflation Reduction Act
+          url: https://www.fisheries.noaa.gov/media-release/biden-harris-administration-announces-34-million-modernize-noaa-fisheries-data
+  singleuser:
+    cloudMetadata:
+      blockWithIptables: false
+    defaultUrl: /lab
+    profileList:
+      - display_name: Default
+        description: Choose image and resource allocation
+        default: true
+        profile_options: &profile_options
+          image: &profile_options_image
+            display_name: Image
+            choices:
+              python:
+                display_name: Py - Openscapes Python 39dffde
+                slug: python
+                kubespawner_override:
+                  image: openscapes/python:39dffde
+              pyrbase:
+                display_name: Py-R - base image 4.4-3.10
+                slug: pyrbase
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/py-rocket-base:latest
+              pyrgeo:
+                display_name: Py-R - Base geospatial image - py-rocket-geospatial latest
+                slug: pyrgeo
+                default: true
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/py-rocket-geospatial:latest
+              coastwatch:
+                display_name: Py-R - CoastWatch - nmfs-opensci coastwatch latest
+                slug: coastwatch
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/coastwatch:latest
+              aomlomics:
+                display_name: Py - Tourmaline Snakemake workflow for QIIME 2 v.2023.5
+                slug: aomlomics
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/aomlomics-jh:latest
+              iorocker:
+                display_name: R - R geospatial w sdmTMB - r-geospatial-sdm latest
+                slug: rgeospatialsdm
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/r-geospatial-sdm:latest
+              echopype:
+                display_name: Py - Echopype with pangeo nmfs-opensci echopype latest
+                slug: echopype
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/echopype:latest
+              arcgis:
+                display_name: Py - ArcGIS Python 3.9
+                slug: arcgis
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/arcgis:latest
+              cboettig:
+                display_name: Py-R - NASA TOPS - boettiger-lab nasa-tops latest
+                slug: cboettig
+                kubespawner_override:
+                  image: ghcr.io/boettiger-lab/nasa-tops:latest
+              vast:
+                display_name: R - VAST with TMB - nmfs-opensci vast latest
+                kubespawner_override:
+                  image: ghcr.io/nmfs-opensci/container-images/vast:latest
+            unlisted_choice:
+              enabled: True
+              display_name: "Custom image"
+              validation_regex: "^.+:.+$"
+              validation_message: "Must be a publicly available docker image, of form <image-name>:<tag>"
+              kubespawner_override:
+                image: "{value}"
+          requests: &profile_options_resource_allocation
+            display_name: Resource Allocation
+            choices:
+              # choices generated by combining:
+              # - deployer generate resource-allocation choices r7i.xlarge
+              # - deployer generate resource-allocation choices r7i.4xlarge --num-allocations=2
+              mem_1_9:
+                display_name: 1.9 GB RAM, upto 3.7 CPUs
+                kubespawner_override:
+                  mem_guarantee: 1991244775
+                  mem_limit: 1991244775
+                  cpu_guarantee: 0.2328125
+                  cpu_limit: 3.725
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.xlarge
+                default: true
+              mem_3_7:
+                display_name: 3.7 GB RAM, upto 3.7 CPUs
+                kubespawner_override:
+                  mem_guarantee: 3982489550
+                  mem_limit: 3982489550
+                  cpu_guarantee: 0.465625
+                  cpu_limit: 3.725
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.xlarge
+              mem_7_4:
+                display_name: 7.4 GB RAM, upto 3.7 CPUs
+                kubespawner_override:
+                  mem_guarantee: 7964979101
+                  mem_limit: 7964979101
+                  cpu_guarantee: 0.93125
+                  cpu_limit: 3.725
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.xlarge
+              mem_14_8:
+                display_name: 14.8 GB RAM, upto 3.7 CPUs
+                kubespawner_override:
+                  mem_guarantee: 15929958203
+                  mem_limit: 15929958203
+                  cpu_guarantee: 1.8625
+                  cpu_limit: 3.725
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.xlarge
+              mem_29_7:
+                display_name: 29.7 GB RAM, upto 3.7 CPUs
+                kubespawner_override:
+                  mem_guarantee: 31859916406
+                  mem_limit: 31859916406
+                  cpu_guarantee: 3.725
+                  cpu_limit: 3.725
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.xlarge
+              mem_60_6:
+                display_name: 60.6 GB RAM, upto 15.6 CPUs
+                kubespawner_override:
+                  mem_guarantee: 65094448840
+                  mem_limit: 65094448840
+                  cpu_guarantee: 7.8475
+                  cpu_limit: 15.695
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.4xlarge
+              mem_121_2:
+                display_name: 121.2 GB RAM, upto 15.6 CPUs
+                kubespawner_override:
+                  mem_guarantee: 130188897681
+                  mem_limit: 130188897681
+                  cpu_guarantee: 15.695
+                  cpu_limit: 15.695
+                  node_selector:
+                    node.kubernetes.io/instance-type: r7i.4xlarge
+  hub:
+    allowNamedServers: true
+    config:
+      JupyterHub:
+        authenticator_class: github
+      GitHubOAuthenticator:
+        populate_teams_in_auth_state: true
+        allowed_organizations:
+          - nmfs-openscapes:longterm-access-2i2c
+        scope:
+          - read:org
+      Authenticator:
+        enable_auth_state: true
+        admin_users:
+          - ateucher # Andy Teucher
+          - jules32 # Julia Stewart Lowndes
+          - eeholmes # Eli Holmes
+
+  scheduling:
+    userScheduler:
+      enabled: true

config/clusters/nmfs-openscapes/enc-prod.secret.values.yaml

@@ -0,0 +1,20 @@
+jupyterhub:
+  hub:
+    config:
+      GitHubOAuthenticator:
+        client_id: ENC[AES256_GCM,data:/kx3ZC05GIXwvPngSXU5bTZlW88=,iv:SysDo0MWDX8L9nJJm9iK0bC6L3sY6rVOruhvWXxxc/U=,tag:wHnei2oDrTs4FYEfUPsrlg==,type:str]
+        client_secret: ENC[AES256_GCM,data:k1/UQcpZy1bqGxNN4GCDfK2Sxi2gOWevayLgc7lVaeQxbAuXlEuBRQ==,iv:KFgWlOqHoYAcNg8CT01euin76bSFQTbE5jFt3pi/VFY=,tag:RvDWgDYxlb47Bmo6IiDJSw==,type:str]
+sops:
+  kms: []
+  gcp_kms:
+    - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+      created_at: "2024-09-10T10:21:22Z"
+      enc: CiUA4OM7eIftbT461fRG4hTcQaNbGW6CXYmRwcmigijBFNx9Qfc0EkkA5dG1Q1Xe5O/suc94v4uBQ7b+F2ZsYZiI8HuR0qpZaY1P1MRUxKQ9YmXcX5AjCg0bEJ52XWnQHq5QV5zn+RvR13sk6yzdAvO+
+  azure_kv: []
+  hc_vault: []
+  age: []
+  lastmodified: "2024-09-10T10:21:56Z"
+  mac: ENC[AES256_GCM,data:fHBKhpc0ehVMu+dgoOr7Mqj0C60OaSxasbYgcZZuQtzKlBeC6GYtdXrIrfA+TBW2RK9l0v9mND/wKDqqX0yI5EvuFzaDRnm6oRw/edL+b2ZJaT1AlY+zTGUDCyOYMbCUQVDZZmfNcMGXUafHG7/IgsV0ogXmQecTT1oiY41xrfs=,iv:T8bjnRSEHaVz7Nl9eUAbipORQhz3GACJfE8DBy+uXzM=,tag:NkE8WnqoFx21bLMzuJL+6Q==,type:str]
+  pgp: []
+  unencrypted_suffix: _unencrypted
+  version: 3.9.0

config/clusters/nmfs-openscapes/enc-staging.secret.values.yaml

@@ -0,0 +1,20 @@
+jupyterhub:
+  hub:
+    config:
+      GitHubOAuthenticator:
+        client_id: ENC[AES256_GCM,data:UaWMGfjMpob7GogQr39sH92bnII=,iv:1MrekKaOXuPDg3Aszrl1bYA32hFCgOrT3swZuy47Ue8=,tag:0ocDXl7/qIMeZvSv5zK+gQ==,type:str]
+        client_secret: ENC[AES256_GCM,data:jj0mqgqGmuiZUPATHvi7RBWjsp/iZl6G2VQ1g0UhTCXnJPfBXL60VA==,iv:ytIS/UNAqB/olTcddfW1GJ2p8FLX+Z2JSLSD6wRYFrk=,tag:O3enlZnXEu6lnpmYZ4EFgA==,type:str]
+sops:
+  kms: []
+  gcp_kms:
+    - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+      created_at: "2024-09-10T10:20:01Z"
+      enc: CiUA4OM7eEJEpHLYCa4g2iDzfz6cz8zZWb0tBSOE1CE4MDDKMa0DEkkA5dG1Q9YSYkBDM3yyqelRpWGsf4AGMg87XKDVGm4cv7+xdTam5S3oLIxeT5IX6O2hBTC9SYuLwNIniEqksX+Q5/s3sM1fb+jq
+  azure_kv: []
+  hc_vault: []
+  age: []
+  lastmodified: "2024-09-10T10:20:43Z"
+  mac: ENC[AES256_GCM,data:i60RLXIx7btXVo05Z+Rqz10MAueF+8AEd3Nz//Oe7rQtWgjNde6HnPD/75YiRXPrDuB9lXiVwn3782x3WYTURHmdP9JUd1FKtdQgNmBIM7896WvshJFC8nUd5rTmizVt2gDD9D3yyH/076XHuBDGqb+LNJIAUmKkwZmEfXUwX10=,iv:JTaL+HEPoMJ482uGcGKNOiRUJ/uipqZkadjcrpSJoXc=,tag:YKkrtk+w4/eWyxH1apMNpw==,type:str]
+  pgp: []
+  unencrypted_suffix: _unencrypted
+  version: 3.9.0

config/clusters/nmfs-openscapes/prod.values.yaml

@@ -0,0 +1,24 @@
+nfs:
+  pv:
+    serverIP: fs-04e46afdb91ce74ae.efs.us-west-2.amazonaws.com
+
+userServiceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::891612562472:role/nmfs-openscapes-prod
+
+jupyterhub:
+  ingress:
+    hosts: [nmfs-openscapes.2i2c.cloud]
+    tls:
+      - hosts: [nmfs-openscapes.2i2c.cloud]
+        secretName: https-auto-tls
+  singleuser:
+    nodeSelector:
+      2i2c.org/hub-name: prod
+    extraEnv:
+      SCRATCH_BUCKET: s3://nmfs-openscapes-scratch/$(JUPYTERHUB_USER)
+      PERSISTENT_BUCKET: s3://nmfs-openscapes-persistent/$(JUPYTERHUB_USER)
+  hub:
+    config:
+      GitHubOAuthenticator:
+        oauth_callback_url: https://nmfs-openscapes.2i2c.cloud/hub/oauth_callback

config/clusters/nmfs-openscapes/staging.values.yaml

@@ -0,0 +1,24 @@
+nfs:
+  pv:
+    serverIP: fs-0bb8ced2e0be85846.efs.us-west-2.amazonaws.com
+
+userServiceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::891612562472:role/nmfs-openscapes-staging
+
+jupyterhub:
+  ingress:
+    hosts: [staging.nmfs-openscapes.2i2c.cloud]
+    tls:
+      - hosts: [staging.nmfs-openscapes.2i2c.cloud]
+        secretName: https-auto-tls
+  singleuser:
+    nodeSelector:
+      2i2c.org/hub-name: staging
+    extraEnv:
+      SCRATCH_BUCKET: s3://nmfs-openscapes-scratch-staging/$(JUPYTERHUB_USER)
+      PERSISTENT_BUCKET: s3://nmfs-openscapes-persistent-staging/$(JUPYTERHUB_USER)
+  hub:
+    config:
+      GitHubOAuthenticator:
+        oauth_callback_url: https://staging.nmfs-openscapes.2i2c.cloud/hub/oauth_callback

eksctl/nmfs-openscapes.jsonnet

@@ -91,10 +91,13 @@ local daskNodes = [];
         [
             {
                 name: "vpc-cni",
+                # FIXME: network policy enforcement doesn't work, what's wrong
+                #        isn't clear.
                 # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
                 configurationValues: |||
                     enableNetworkPolicy: "true"
                 |||,
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
             },
             { name: "coredns" },
             { name: "kube-proxy" },

eksctl/template.jsonnet

@@ -89,10 +89,13 @@ local daskNodes = [];
         [
             {
                 name: "vpc-cni",
+                # FIXME: network policy enforcement doesn't work, what's wrong
+                #        isn't clear.
                 # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
                 configurationValues: |||
                     enableNetworkPolicy: "true"
                 |||,
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
             },
             { name: "coredns" },
             { name: "kube-proxy" },