From 2a4cf18fffbb86fc2110d8aec83204ab8c1b200d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20Fern=C3=A1ndez?= Date: Thu, 4 Jan 2024 14:46:20 +0000 Subject: [PATCH] D4Science fixes (#115) * Make the use of the sidecar optional * Fix linter * Fix code to work with newer oauthenticator * Upgrade dependencies * Bring ophidia configuration in Also refactor a bit --- Dockerfile | 2 +- egi_notebooks_hub/d4science.py | 118 ++++++++++++++++++++++++--------- requirements.txt | 6 +- 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 939b04c..44ba077 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Starting with the image used in helm jupyterhub -FROM jupyterhub/k8s-hub:2.0.0 +FROM quay.io/jupyterhub/k8s-hub:3.2.1 USER root diff --git a/egi_notebooks_hub/d4science.py b/egi_notebooks_hub/d4science.py index 67eafc2..b8340fe 100644 --- a/egi_notebooks_hub/d4science.py +++ b/egi_notebooks_hub/d4science.py @@ -14,7 +14,7 @@ from oauthenticator.oauth2 import OAuthLoginHandler from tornado import web from tornado.httpclient import AsyncHTTPClient, HTTPError, HTTPRequest -from traitlets import Dict, List, Unicode +from traitlets import Bool, Dict, List, Unicode D4SCIENCE_REGISTRY_BASE_URL = os.environ.get( "D4SCIENCE_REGISTRY_BASE_URL", @@ -38,10 +38,10 @@ class D4ScienceContextHandler(OAuthLoginHandler): - def get_state(self): + def get(self): context = self.get_argument("context", None) self.authenticator.d4science_context = context - return super().get_state() + return super().get() class D4ScienceOauthenticator(GenericOAuthenticator): @@ -250,6 +250,40 @@ class D4ScienceSpawner(KubeSpawner): config=True, help="""Frame ancestors for embedding the hub in d4science""", ) + use_ophidia = Bool( + True, + config=True, + help="""Whether to enable or not the ophidia setup""", + ) + ophidia_image = Unicode( + "ophidiabigdata/ophidia-backend-hub:v1.1", + config=True, + help="""Ophidia image""", + ) + ophidia_user = Unicode( + "oph-test", + config=True, + help="""Ophidia user""", + ) + ophidia_passwd = Unicode( + "abcd", + config=True, + help="""Ophidia password""", + ) + workspace_security_context = Dict( + { + "capabilities": {"add": ["SYS_ADMIN"]}, + "privileged": True, + "runAsUser": 1000, + }, + config=True, + help="""Container security context for mounting the workspace""", + ) + use_sidecar = Bool( + True, + config=True, + help="""Whether to use or not a sidecar for the workspace""", + ) sidecar_image = Unicode( "eginotebooks/d4science-storage", config=True, @@ -317,7 +351,7 @@ def get_args(self): } # TODO: check if this keeps making sense return [ - "--SingleUserNotebookApp.tornado_settings=%s" % tornado_settings, + "--ServerApp.tornado_settings=%s" % tornado_settings, "--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'", "--FileContentsManager.use_atomic_writing=False", "--ResourceUseDisplay.track_cpu_percent=True", @@ -449,9 +483,53 @@ def profile_list(self, spawner): self.log.debug("Profiles: %s", sorted_profiles) return sorted_profiles - async def pre_spawn_hook(self, spawner): - # add volumes as defined in the D4Science info sys + def _configure_ophidia(self, spawner): + if not self.use_ophidia: + return + chosen_profile = spawner.user_options.get("profile", "") + if "ophidia" in chosen_profile: + ophidia_mounts = [ + m for m in spawner.volume_mounts if m["name"] != "workspace" + ] + ophidia = { + "name": "ophidia", + "image": self.ophidia_image, + "volumeMounts": ophidia_mounts, + } + spawner.extra_containers.append(ophidia) + spawner.environment["OPH_USER"] = self.ophidia_user + spawner.environment["OPH_PASSWD"] = self.ophidia_passwd + spawner.environment["OPH_SERVER_HOST"] = "127.0.0.1" + spawner.environment["HDF5_USE_FILE_LOCKING"] = "FALSE" + + def _configure_workspace(self, spawner): token = spawner.environment.get("D4SCIENCE_TOKEN", "") + if not token: + self.log.debug("Not configuring workspace access, there is no token") + return + if self.use_sidecar: + sidecar = { + "name": "workspace-sidecar", + "image": self.sidecar_image, + "securityContext": self.workspace_security_context, + "env": [ + {"name": "MNTPATH", "value": "/workspace"}, + {"name": "D4SCIENCE_TOKEN", "value": token}, + ], + "volumeMounts": [ + {"mountPath": "/workspace:shared", "name": "workspace"}, + ], + "lifecycle": { + "preStop": { + "exec": {"command": ["fusermount", "-uz", "/workspace"]} + }, + }, + } + spawner.extra_containers.append(sidecar) + else: + spawner.container_security_context = self.workspace_security_context + + async def pre_spawn_hook(self, spawner): context = spawner.environment.get("D4SCIENCE_CONTEXT", "") if context: # set the whole context as annotation (needed for accounting) @@ -460,27 +538,7 @@ async def pre_spawn_hook(self, spawner): vre = context[context.rindex("/") + 1 :] spawner.log.debug("VRE: %s", vre) spawner.environment["VRE"] = vre - if token: - spawner.extra_containers = [ - { - "name": "workspace-sidecar", - "image": self.sidecar_image, - "securityContext": { - "privileged": True, - "capabilities": {"add": ["SYS_ADMIN"]}, - "runAsUser": 1000, - }, - "env": [ - {"name": "MNTPATH", "value": "/workspace"}, - {"name": "D4SCIENCE_TOKEN", "value": token}, - ], - "volumeMounts": [ - {"mountPath": "/workspace:shared", "name": "workspace"}, - ], - "lifecycle": { - "preStop": { - "exec": {"command": ["fusermount", "-uz", "/workspace"]} - }, - }, - } - ] + # TODO(enolfc): check whether assigning to [] is safe + spawner.extra_containers = [] + self._configure_workspace(spawner) + self._configure_ophidia(spawner) diff --git a/requirements.txt b/requirements.txt index 853466c..78f7c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -jupyterhub>=1.2 -oauthenticator>=14.0.0 -jupyterhub-kubespawner>=4.3.0 +jupyterhub>=4.0.2 +oauthenticator>=16.1.0 +jupyterhub-kubespawner>=6.1.0 xmltodict