Merge pull request #4404 from GeorgianaElena/binderhub-selectors

Spin up binder build pods and user nodes on smallest nodes

config/clusters/2i2c/binderhub-ui-demo.values.yaml

@@ -33,6 +33,10 @@ jupyterhub:
     singleuserAdmin:
       extraVolumeMounts: []
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
     storage:
       type: none
       extraVolumeMounts: []
@@ -68,6 +72,11 @@ jupyterhub:
               username_claim: "email"
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
   ingress:
     enabled: true
     hosts:
@@ -77,6 +86,11 @@ binderhub-service:
         hosts:
           - binderhub-ui-demo.2i2c.cloud
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: n2-highmem-4
     BinderHub:
       base_url: /
       hub_url: https://hub.binderhub-ui-demo.2i2c.cloud

config/clusters/hhmi/binder.values.yaml

@@ -20,6 +20,10 @@ jupyterhub:
       templateVars:
         enabled: false
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
     cloudMetadata:
       blockWithIptables: false
     cpu:
@@ -81,13 +85,23 @@ jupyterhub:
           - access:services!service=binder
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
   ingress:
     enabled: true
     hosts: [binder.hhmi.2i2c.cloud]
     tls:
       - hosts: [binder.hhmi.2i2c.cloud]
         secretName: binder-https-auto-tls
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: n2-highmem-4
     BinderHub:
       base_url: /
       hub_url: https://hub.binder.hhmi.2i2c.cloud

config/clusters/opensci/big-binder.values.yaml

@@ -48,6 +48,10 @@ jupyterhub:
     singleuserAdmin:
       extraVolumeMounts: []
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
     cloudMetadata:
       blockWithIptables: false
     cpu:
@@ -89,13 +93,23 @@ jupyterhub:
           - access:services!service=binder
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
   ingress:
     enabled: true
     hosts: [big.binder.opensci.2i2c.cloud]
     tls:
       - hosts: [big.binder.opensci.2i2c.cloud]
         secretName: binder-https-auto-tls
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: r5.xlarge
     BinderHub:
       base_url: /
       hub_url: https://hub.big.binder.opensci.2i2c.cloud

config/clusters/opensci/small-binder.values.yaml

@@ -17,6 +17,10 @@ jupyterhub:
       templateVars:
         enabled: false
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
     cpu:
       limit: 2
     memory:
@@ -76,13 +80,23 @@ jupyterhub:
           - access:services!service=binder
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
   ingress:
     enabled: true
     hosts: [binder.opensci.2i2c.cloud]
     tls:
       - hosts: [binder.opensci.2i2c.cloud]
         secretName: binder-https-auto-tls
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: r5.xlarge
     BinderHub:
       base_url: /
       hub_url: https://hub.binder.opensci.2i2c.cloud

config/clusters/projectpythia/pythia-binder.values.yaml

@@ -16,6 +16,10 @@ jupyterhub:
     singleuserAdmin:
       extraVolumeMounts: []
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
     storage:
       type: none
       extraVolumeMounts: []
@@ -74,13 +78,23 @@ jupyterhub:
           - access:services!service=binder
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: r5.xlarge
   ingress:
     enabled: true
     hosts: [binder.pythia.2i2c.cloud]
     tls:
       - hosts: [binder.pythia.2i2c.cloud]
         secretName: binder-https-auto-tls
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: r5.xlarge
     GitHubRepoProvider:
       allowed_specs:
         - ^ProjectPythia/.*$

config/clusters/templates/common/binderhub-ui-hub.values.yaml

@@ -35,6 +35,14 @@ jupyterhub:
     singleuserAdmin:
       extraVolumeMounts: []
   singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      {% if provider == "gcp" %}
+      node.kubernetes.io/instance-type: n2-highmem-4
+      {% else %}
+      node.kubernetes.io/instance-type: r5.xlarge
+      {% endif %}
     storage:
       type: none
       extraVolumeMounts: []
@@ -119,13 +127,31 @@ jupyterhub:
           - access:services!service=binder
 binderhub-service:
   enabled: true
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      {% if provider == "gcp" %}
+      node.kubernetes.io/instance-type: n2-highmem-4
+      {% else %}
+      node.kubernetes.io/instance-type: r5.xlarge
+      {% endif %}
   ingress:
     enabled: true
     hosts: [{{ binderhub_domain }}]
     tls:
       - hosts: [{{ binderhub_domain }}]
         secretName: binder-https-auto-tls
   config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        {% if provider == "gcp" %}
+        node.kubernetes.io/instance-type: n2-highmem-4
+        {% else %}
+        node.kubernetes.io/instance-type: r5.xlarge
+        {% endif %}
     BinderHub:
       base_url: /
       hub_url: https://{{ jupyterhub_domain }}

deployer/commands/generate/hub_asset/hub_files.py

@@ -86,6 +86,9 @@ def binderhub_ui_values_file(
     cluster_name: str = typer.Option(
         ..., prompt="Name of the cluster where the hub will live"
     ),
+    provider: str = typer.Option(
+        ..., prompt="Name of the provider where the hub will live (gcp/aws)"
+    ),
     jupyterhub_domain: str = typer.Option(
         ...,
         prompt="Domain where jupyterhub will run (ex. hub.binder.community.2i2c.cloud)",
@@ -127,6 +130,7 @@ def binderhub_ui_values_file(
 
     vars = {
         "cluster_name": cluster_name,
+        "provider": provider,
         "jupyterhub_domain": jupyterhub_domain,
         "binderhub_domain": binderhub_domain,
         "authenticator": authenticator,

docs/howto/features/binderhub-ui.md

@@ -68,7 +68,7 @@ Some of the configuration that gets inherited either from the `basehub` defaults
         initContainers: []
         profileList: []
     ```
-#### 2. Check jupyterhub and binderhub domains setup
+#### 1. Check jupyterhub and binderhub domains setup
 
 Having separate domains for both jupyterhub and binderhub will help with having clean and correct sharing URLs without having them be based off the `hub/services/:name` path.
 
@@ -94,7 +94,7 @@ binderhub-service:
         secretName: binder-https-auto-tls
 ```
 
-#### 3. Check that binderhubUI is enabled
+#### 1. Check that binderhubUI is enabled
 
 Enable `jupyterhub.custom.binderhubUI` which will in turn enable the hub to use [BinderSpawnerMixin](https://github.com/jupyterhub/binderhub/blob/bd297b2c3f713cf46b0b22cfabc86d8140bbed41/helm-chart/binderhub/values.yaml#L115-L207) that allows converting JupyterHub container spawners into BinderHub spawners
 
@@ -105,7 +105,7 @@ jupyterhub:
       enabled: true
 ```
 
-#### 4. Check that the binderhub-service chart is enabled
+#### 1. Check that the binderhub-service chart is enabled
 
 We will use the [binderhub-service](https://github.com/2i2c-org/binderhub-service/) Helm chart to run BinderHub, the Python software, as a standalone service to build and push images with [repo2docker](https://github.com/jupyterhub/repo2docker), next to JupyterHub so we need to enable it.
 
@@ -114,7 +114,7 @@ binderhub-service:
   enabled: true
 ```
 
-#### 5. Check that BinderHub is configured correctly
+#### 1. Check that BinderHub is configured correctly
 
 We need to configure BinderHub so that:
 
@@ -131,7 +131,33 @@ binderhub-service:
       enable_api_only_mode: false
 ```
 
-#### 6. Check the binderhub extra env variables
+#### 1. Check that the builder docker api and user pods are scheduled on the smallest available instance
+
+In general, for GCP, they should run on `n2-highmem-4` and on AWS they should be placed on `r5.xlarge` machines. But it's best to double-check the cluster's terraform or eksctl configuration files to make sure this is the smallest instance and not another one.
+
+
+```yaml
+binderhub-service:
+  dockerApi:
+    nodeSelector:
+      # Schedule dockerApi pods to run on the smallest user nodes only
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
+  config:
+    KubernetesBuildExecutor:
+      nodeSelector:
+        # Schedule builder pods to run on the smallest user nodes only
+        # https://github.com/2i2c-org/infrastructure/issues/4241
+        node.kubernetes.io/instance-type: n2-highmem-4
+jupyterhub:
+  singleuser:
+    nodeSelector:
+      # Schedule users on the smallest instance
+      # https://github.com/2i2c-org/infrastructure/issues/4241
+      node.kubernetes.io/instance-type: n2-highmem-4
+```
+
+#### 1. Check the binderhub extra env variables
 
 These are needed by the jupyterhub software bits that the binderhub software uses.