From 27f5ab285e0b18c4c32fa0d623a486df5d1fc29e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 10 Sep 2024 12:38:11 +0200 Subject: [PATCH 1/2] nmfs-openscapes: add staging and prod hubs --- .github/workflows/deploy-hubs.yaml | 1 + config/clusters/nmfs-openscapes/cluster.yaml | 29 +-- .../nmfs-openscapes/common.values.yaml | 197 ++++++++++++++++++ .../enc-prod.secret.values.yaml | 20 ++ .../enc-staging.secret.values.yaml | 20 ++ .../clusters/nmfs-openscapes/prod.values.yaml | 24 +++ .../nmfs-openscapes/staging.values.yaml | 24 +++ 7 files changed, 302 insertions(+), 13 deletions(-) create mode 100644 config/clusters/nmfs-openscapes/common.values.yaml create mode 100644 config/clusters/nmfs-openscapes/enc-prod.secret.values.yaml create mode 100644 config/clusters/nmfs-openscapes/enc-staging.secret.values.yaml create mode 100644 config/clusters/nmfs-openscapes/prod.values.yaml create mode 100644 config/clusters/nmfs-openscapes/staging.values.yaml diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index d79bb7d46..73f13b9a2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -245,6 +245,7 @@ jobs: failure_nasa-esdis: "${{ env.failure_nasa-esdis }}" failure_nasa-ghg: "${{ env.failure_nasa-ghg }}" failure_nasa-veda: "${{ env.failure_nasa-veda }}" + failure_nmfs-openscapes: "${{ env.failure_nmfs-openscapes }}" failure_openscapes: "${{ env.failure_openscapes }}" failure_opensci: "${{ env.failure_opensci }}" failure_pangeo-hubs: "${{ env.failure_pangeo-hubs }}" diff --git a/config/clusters/nmfs-openscapes/cluster.yaml b/config/clusters/nmfs-openscapes/cluster.yaml index 0ce88298a..55b6c39ee 100644 --- a/config/clusters/nmfs-openscapes/cluster.yaml +++ b/config/clusters/nmfs-openscapes/cluster.yaml @@ -12,16 +12,19 @@ support: - support.values.yaml - enc-support.secret.values.yaml hubs: - [] - # Uncomment the lines below once the support infrastructure was deployed and - # you are ready to add the first cluster - - # - name: - # # Tip: consider changing this to something more human friendly - # display_name: "nmfs-openscapes - " - # domain: .nmfs-openscapes.2i2c.cloud - # helm_chart: basehub - # helm_chart_values_files: - # - common.values.yaml - # - .values.yaml - # - enc-.secret.values.yaml + - name: staging + display_name: NOAA Fisheries Openscapes - Staging + domain: staging.nmfs-openscapes.2i2c.cloud + helm_chart: basehub + helm_chart_values_files: + - common.values.yaml + - staging.values.yaml + - enc-staging.secret.values.yaml + - name: prod + display_name: NOAA Fisheries Openscapes + domain: nmfs-openscapes.2i2c.cloud + helm_chart: basehub + helm_chart_values_files: + - common.values.yaml + - staging.values.yaml + - enc-staging.secret.values.yaml diff --git a/config/clusters/nmfs-openscapes/common.values.yaml b/config/clusters/nmfs-openscapes/common.values.yaml new file mode 100644 index 000000000..d1c759b18 --- /dev/null +++ b/config/clusters/nmfs-openscapes/common.values.yaml @@ -0,0 +1,197 @@ +nfs: + enabled: true + pv: + enabled: true + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport + baseShareName: / + # serverIP is set in staging / prod respectively + +jupyterhub: + custom: + 2i2c: + add_staff_user_ids_to_admin_users: true + add_staff_user_ids_of_type: github + jupyterhubConfigurator: + enabled: false + homepage: + templateVars: + org: + name: NOAA Fisheries Openscapes + logo_url: https://raw.githubusercontent.com/nmfs-openscapes/nmfs-openscapes.github.io/main/images/logo-transparent-crop.png + url: https://nmfs-openscapes.github.io/ + designed_by: + name: 2i2c + url: https://2i2c.org + operated_by: + name: 2i2c + url: https://2i2c.org + funded_by: + name: NOAA Fisheries and the Biden-Harris Administration's Inflation Reduction Act + url: https://www.fisheries.noaa.gov/media-release/biden-harris-administration-announces-34-million-modernize-noaa-fisheries-data + singleuser: + cloudMetadata: + blockWithIptables: false + defaultUrl: /lab + profileList: + - display_name: Default + description: Choose image and resource allocation + default: true + profile_options: &profile_options + image: &profile_options_image + display_name: Image + choices: + python: + display_name: Py - Openscapes Python 39dffde + slug: python + kubespawner_override: + image: openscapes/python:39dffde + pyrbase: + display_name: Py-R - base image 4.4-3.10 + slug: pyrbase + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/py-rocket-base:latest + pyrgeo: + display_name: Py-R - Base geospatial image - py-rocket-geospatial latest + slug: pyrgeo + default: true + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/py-rocket-geospatial:latest + coastwatch: + display_name: Py-R - CoastWatch - nmfs-opensci coastwatch latest + slug: coastwatch + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/coastwatch:latest + aomlomics: + display_name: Py - Tourmaline Snakemake workflow for QIIME 2 v.2023.5 + slug: aomlomics + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/aomlomics-jh:latest + iorocker: + display_name: R - R geospatial w sdmTMB - r-geospatial-sdm latest + slug: rgeospatialsdm + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/r-geospatial-sdm:latest + echopype: + display_name: Py - Echopype with pangeo nmfs-opensci echopype latest + slug: echopype + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/echopype:latest + arcgis: + display_name: Py - ArcGIS Python 3.9 + slug: arcgis + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/arcgis:latest + cboettig: + display_name: Py-R - NASA TOPS - boettiger-lab nasa-tops latest + slug: cboettig + kubespawner_override: + image: ghcr.io/boettiger-lab/nasa-tops:latest + vast: + display_name: R - VAST with TMB - nmfs-opensci vast latest + kubespawner_override: + image: ghcr.io/nmfs-opensci/container-images/vast:latest + unlisted_choice: + enabled: True + display_name: "Custom image" + validation_regex: "^.+:.+$" + validation_message: "Must be a publicly available docker image, of form :" + kubespawner_override: + image: "{value}" + requests: &profile_options_resource_allocation + display_name: Resource Allocation + choices: + # choices generated by combining: + # - deployer generate resource-allocation choices r7i.xlarge + # - deployer generate resource-allocation choices r7i.4xlarge --num-allocations=2 + mem_1_9: + display_name: 1.9 GB RAM, upto 3.7 CPUs + kubespawner_override: + mem_guarantee: 1991244775 + mem_limit: 1991244775 + cpu_guarantee: 0.2328125 + cpu_limit: 3.725 + node_selector: + node.kubernetes.io/instance-type: r7i.xlarge + default: true + mem_3_7: + display_name: 3.7 GB RAM, upto 3.7 CPUs + kubespawner_override: + mem_guarantee: 3982489550 + mem_limit: 3982489550 + cpu_guarantee: 0.465625 + cpu_limit: 3.725 + node_selector: + node.kubernetes.io/instance-type: r7i.xlarge + mem_7_4: + display_name: 7.4 GB RAM, upto 3.7 CPUs + kubespawner_override: + mem_guarantee: 7964979101 + mem_limit: 7964979101 + cpu_guarantee: 0.93125 + cpu_limit: 3.725 + node_selector: + node.kubernetes.io/instance-type: r7i.xlarge + mem_14_8: + display_name: 14.8 GB RAM, upto 3.7 CPUs + kubespawner_override: + mem_guarantee: 15929958203 + mem_limit: 15929958203 + cpu_guarantee: 1.8625 + cpu_limit: 3.725 + node_selector: + node.kubernetes.io/instance-type: r7i.xlarge + mem_29_7: + display_name: 29.7 GB RAM, upto 3.7 CPUs + kubespawner_override: + mem_guarantee: 31859916406 + mem_limit: 31859916406 + cpu_guarantee: 3.725 + cpu_limit: 3.725 + node_selector: + node.kubernetes.io/instance-type: r7i.xlarge + mem_60_6: + display_name: 60.6 GB RAM, upto 15.6 CPUs + kubespawner_override: + mem_guarantee: 65094448840 + mem_limit: 65094448840 + cpu_guarantee: 7.8475 + cpu_limit: 15.695 + node_selector: + node.kubernetes.io/instance-type: r7i.4xlarge + mem_121_2: + display_name: 121.2 GB RAM, upto 15.6 CPUs + kubespawner_override: + mem_guarantee: 130188897681 + mem_limit: 130188897681 + cpu_guarantee: 15.695 + cpu_limit: 15.695 + node_selector: + node.kubernetes.io/instance-type: r7i.4xlarge + hub: + allowNamedServers: true + config: + JupyterHub: + authenticator_class: github + GitHubOAuthenticator: + populate_teams_in_auth_state: true + allowed_organizations: + - nmfs-openscapes:longterm-access-2i2c + scope: + - read:org + Authenticator: + enable_auth_state: true + admin_users: + - ateucher # Andy Teucher + - jules32 # Julia Stewart Lowndes + - eeholmes # Eli Holmes + + scheduling: + userScheduler: + enabled: true diff --git a/config/clusters/nmfs-openscapes/enc-prod.secret.values.yaml b/config/clusters/nmfs-openscapes/enc-prod.secret.values.yaml new file mode 100644 index 000000000..963ed112e --- /dev/null +++ b/config/clusters/nmfs-openscapes/enc-prod.secret.values.yaml @@ -0,0 +1,20 @@ +jupyterhub: + hub: + config: + GitHubOAuthenticator: + client_id: ENC[AES256_GCM,data:/kx3ZC05GIXwvPngSXU5bTZlW88=,iv:SysDo0MWDX8L9nJJm9iK0bC6L3sY6rVOruhvWXxxc/U=,tag:wHnei2oDrTs4FYEfUPsrlg==,type:str] + client_secret: ENC[AES256_GCM,data:k1/UQcpZy1bqGxNN4GCDfK2Sxi2gOWevayLgc7lVaeQxbAuXlEuBRQ==,iv:KFgWlOqHoYAcNg8CT01euin76bSFQTbE5jFt3pi/VFY=,tag:RvDWgDYxlb47Bmo6IiDJSw==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2024-09-10T10:21:22Z" + enc: CiUA4OM7eIftbT461fRG4hTcQaNbGW6CXYmRwcmigijBFNx9Qfc0EkkA5dG1Q1Xe5O/suc94v4uBQ7b+F2ZsYZiI8HuR0qpZaY1P1MRUxKQ9YmXcX5AjCg0bEJ52XWnQHq5QV5zn+RvR13sk6yzdAvO+ + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-09-10T10:21:56Z" + mac: ENC[AES256_GCM,data:fHBKhpc0ehVMu+dgoOr7Mqj0C60OaSxasbYgcZZuQtzKlBeC6GYtdXrIrfA+TBW2RK9l0v9mND/wKDqqX0yI5EvuFzaDRnm6oRw/edL+b2ZJaT1AlY+zTGUDCyOYMbCUQVDZZmfNcMGXUafHG7/IgsV0ogXmQecTT1oiY41xrfs=,iv:T8bjnRSEHaVz7Nl9eUAbipORQhz3GACJfE8DBy+uXzM=,tag:NkE8WnqoFx21bLMzuJL+6Q==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.0 diff --git a/config/clusters/nmfs-openscapes/enc-staging.secret.values.yaml b/config/clusters/nmfs-openscapes/enc-staging.secret.values.yaml new file mode 100644 index 000000000..1ae04aba1 --- /dev/null +++ b/config/clusters/nmfs-openscapes/enc-staging.secret.values.yaml @@ -0,0 +1,20 @@ +jupyterhub: + hub: + config: + GitHubOAuthenticator: + client_id: ENC[AES256_GCM,data:UaWMGfjMpob7GogQr39sH92bnII=,iv:1MrekKaOXuPDg3Aszrl1bYA32hFCgOrT3swZuy47Ue8=,tag:0ocDXl7/qIMeZvSv5zK+gQ==,type:str] + client_secret: ENC[AES256_GCM,data:jj0mqgqGmuiZUPATHvi7RBWjsp/iZl6G2VQ1g0UhTCXnJPfBXL60VA==,iv:ytIS/UNAqB/olTcddfW1GJ2p8FLX+Z2JSLSD6wRYFrk=,tag:O3enlZnXEu6lnpmYZ4EFgA==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2024-09-10T10:20:01Z" + enc: CiUA4OM7eEJEpHLYCa4g2iDzfz6cz8zZWb0tBSOE1CE4MDDKMa0DEkkA5dG1Q9YSYkBDM3yyqelRpWGsf4AGMg87XKDVGm4cv7+xdTam5S3oLIxeT5IX6O2hBTC9SYuLwNIniEqksX+Q5/s3sM1fb+jq + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-09-10T10:20:43Z" + mac: ENC[AES256_GCM,data:i60RLXIx7btXVo05Z+Rqz10MAueF+8AEd3Nz//Oe7rQtWgjNde6HnPD/75YiRXPrDuB9lXiVwn3782x3WYTURHmdP9JUd1FKtdQgNmBIM7896WvshJFC8nUd5rTmizVt2gDD9D3yyH/076XHuBDGqb+LNJIAUmKkwZmEfXUwX10=,iv:JTaL+HEPoMJ482uGcGKNOiRUJ/uipqZkadjcrpSJoXc=,tag:YKkrtk+w4/eWyxH1apMNpw==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.0 diff --git a/config/clusters/nmfs-openscapes/prod.values.yaml b/config/clusters/nmfs-openscapes/prod.values.yaml new file mode 100644 index 000000000..2c355b2ad --- /dev/null +++ b/config/clusters/nmfs-openscapes/prod.values.yaml @@ -0,0 +1,24 @@ +nfs: + pv: + serverIP: fs-04e46afdb91ce74ae.efs.us-west-2.amazonaws.com + +userServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::891612562472:role/nmfs-openscapes-prod + +jupyterhub: + ingress: + hosts: [nmfs-openscapes.2i2c.cloud] + tls: + - hosts: [nmfs-openscapes.2i2c.cloud] + secretName: https-auto-tls + singleuser: + nodeSelector: + 2i2c.org/hub-name: prod + extraEnv: + SCRATCH_BUCKET: s3://nmfs-openscapes-scratch/$(JUPYTERHUB_USER) + PERSISTENT_BUCKET: s3://nmfs-openscapes-persistent/$(JUPYTERHUB_USER) + hub: + config: + GitHubOAuthenticator: + oauth_callback_url: https://nmfs-openscapes.2i2c.cloud/hub/oauth_callback diff --git a/config/clusters/nmfs-openscapes/staging.values.yaml b/config/clusters/nmfs-openscapes/staging.values.yaml new file mode 100644 index 000000000..9ad9b9a2f --- /dev/null +++ b/config/clusters/nmfs-openscapes/staging.values.yaml @@ -0,0 +1,24 @@ +nfs: + pv: + serverIP: fs-0bb8ced2e0be85846.efs.us-west-2.amazonaws.com + +userServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::891612562472:role/nmfs-openscapes-staging + +jupyterhub: + ingress: + hosts: [staging.nmfs-openscapes.2i2c.cloud] + tls: + - hosts: [staging.nmfs-openscapes.2i2c.cloud] + secretName: https-auto-tls + singleuser: + nodeSelector: + 2i2c.org/hub-name: staging + extraEnv: + SCRATCH_BUCKET: s3://nmfs-openscapes-scratch-staging/$(JUPYTERHUB_USER) + PERSISTENT_BUCKET: s3://nmfs-openscapes-persistent-staging/$(JUPYTERHUB_USER) + hub: + config: + GitHubOAuthenticator: + oauth_callback_url: https://staging.nmfs-openscapes.2i2c.cloud/hub/oauth_callback From c7d530e2a4f69a513b1d7189a39939e5d6984703 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 10 Sep 2024 14:48:05 +0200 Subject: [PATCH 2/2] eksctl: add note about failing netpol enforcement --- eksctl/nmfs-openscapes.jsonnet | 3 +++ eksctl/template.jsonnet | 3 +++ 2 files changed, 6 insertions(+) diff --git a/eksctl/nmfs-openscapes.jsonnet b/eksctl/nmfs-openscapes.jsonnet index 62889cac2..2325051e9 100644 --- a/eksctl/nmfs-openscapes.jsonnet +++ b/eksctl/nmfs-openscapes.jsonnet @@ -91,10 +91,13 @@ local daskNodes = []; [ { name: "vpc-cni", + # FIXME: network policy enforcement doesn't work, what's wrong + # isn't clear. # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml configurationValues: ||| enableNetworkPolicy: "true" |||, + attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"], }, { name: "coredns" }, { name: "kube-proxy" }, diff --git a/eksctl/template.jsonnet b/eksctl/template.jsonnet index 7a9b83c2c..5837f860d 100644 --- a/eksctl/template.jsonnet +++ b/eksctl/template.jsonnet @@ -89,10 +89,13 @@ local daskNodes = []; [ { name: "vpc-cni", + # FIXME: network policy enforcement doesn't work, what's wrong + # isn't clear. # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml configurationValues: ||| enableNetworkPolicy: "true" |||, + attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"], }, { name: "coredns" }, { name: "kube-proxy" },