From 3ec4322723d6fa63297e07621d03f13569062336 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 16 Sep 2024 20:33:00 +0100 Subject: [PATCH] Deploy app from git repository (#445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * get app config from git * add documentation for getting app config from git * remove unused vars * add git url in spawner * add example * add positive and neg tests * move app from repo test to e2e section * add test for invalid repository * improve description * git repo branch add to user options * add test for create server from git repo * replace aktech with jovyan * use conda-project instead * fix linting * move conda project import to inside function * remove unnecessary config file * app config from conda project * log message on failure * move app from git to separate module * move app from git to a separate directory * add more docs * add test to parse jhub-app config from conda-project yml * use repository object for server creation * update create server with git repo test * add integration tests for api * set oauth_no_confirm=True * remove clicking on authorize * always create artifact name * extract common items for user options * add a line about cloning repository * add another suffix to app name * Apply suggestions from code review Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com> --------- Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com> --- .github/workflows/test-integration.yml | 1 + environment-dev.yml | 2 + jhub_apps/service/app_from_git.py | 125 ++++++++++++++++++ jhub_apps/service/models.py | 23 +++- jhub_apps/service/routes.py | 21 +++ jhub_apps/spawner/spawner_creation.py | 14 ++ jhub_apps/tests/common/constants.py | 3 + jhub_apps/tests/tests_e2e/test_api.py | 106 +++++++++++++++ jhub_apps/tests/tests_e2e/test_integration.py | 2 - jhub_apps/tests/tests_e2e/utils.py | 60 +++++++++ jhub_apps/tests/tests_unit/test_api.py | 30 +++++ .../tests/tests_unit/test_app_from_git.py | 31 +++++ jupyterhub_config.py | 2 +- pyproject.toml | 4 + 14 files changed, 415 insertions(+), 9 deletions(-) create mode 100644 jhub_apps/service/app_from_git.py create mode 100644 jhub_apps/tests/tests_e2e/utils.py create mode 100644 jhub_apps/tests/tests_unit/test_app_from_git.py diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index f6fe391d..644310dc 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -92,6 +92,7 @@ jobs: - name: Create artifact name id: artifact-name + if: always() run: | if [ "${{ matrix.jupyterhub }}" = "4.1.5" ]; then jhub_suffix="4x" diff --git a/environment-dev.yml b/environment-dev.yml index 0186e2ce..93d76968 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -22,3 +22,5 @@ dependencies: - cachetools - structlog - gradio + - gitpython + - conda-project=0.4.2 diff --git a/jhub_apps/service/app_from_git.py b/jhub_apps/service/app_from_git.py new file mode 100644 index 00000000..6dd7dda7 --- /dev/null +++ b/jhub_apps/service/app_from_git.py @@ -0,0 +1,125 @@ +import os +import pathlib +import tempfile +from pathlib import Path + +import git +from fastapi import HTTPException, status +from pydantic import ValidationError + +from jhub_apps.service.models import Repository, JHubAppConfig +from jhub_apps.service.utils import logger, encode_file_to_data_url + + +def _clone_repo(repository: Repository, temp_dir): + """Clone repository to the given tem_dir""" + try: + logger.info("Trying to clone repository", repo_url=repository.url) + git.Repo.clone_from(repository.url, temp_dir, depth=1, branch=repository.ref) + except Exception as e: + message = f"Repository clone failed: {repository.url}" + logger.error(message, repo_url=repository.url) + logger.error(e) + raise HTTPException( + detail=message, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + +def _get_app_configuration_from_git( + repository: Repository +) -> JHubAppConfig: + """Clones the git directory into a temporary path and extracts all the metadata + about the app from conda-project's config yaml. + """ + with tempfile.TemporaryDirectory() as temp_dir: + _clone_repo(repository, temp_dir) + _check_conda_project_config_directory_exists(repository, temp_dir) + conda_project_yaml = _get_conda_project_config_yaml(temp_dir) + jhub_apps_config_dict = _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml) + app_config = _load_jhub_app_config_to_pydantic_model( + jhub_apps_config_dict, + repository, + temp_dir + ) + return app_config + + +def _load_jhub_app_config_to_pydantic_model( + jhub_apps_config_dict: dict, repository: Repository, temp_dir: str +): + """Load the parsed jhub-apps config into pydantic model for validation""" + thumbnail_base64 = "" + thumbnail_path_from_config = jhub_apps_config_dict.get("thumbnail_path") + if thumbnail_path_from_config: + thumbnail_path = Path(os.path.join(temp_dir, thumbnail_path_from_config)) + thumbnail_base64 = encode_file_to_data_url( + filename=thumbnail_path.name, file_contents=thumbnail_path.read_bytes() + ) + try: + # Load YAML content into the Pydantic model + app_config = JHubAppConfig(**{ + **jhub_apps_config_dict, + "repository": repository, + "thumbnail": thumbnail_base64, + "env": jhub_apps_config_dict.get("environment", {}) + }) + except ValidationError as e: + message = f"Validation error: {e}" + logger.error(message) + raise HTTPException( + detail=message, + status_code=status.HTTP_400_BAD_REQUEST, + ) + return app_config + + +def _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml): + """Extracts jhub-apps app config from conda project yaml's config""" + jhub_apps_variables = { + k.split("JHUB_APP_CONFIG_")[-1]: v for k, v in conda_project_yaml.variables.items() + if k.startswith("JHUB_APP_CONFIG_") + } + environment_variables = { + k: v for k, v in conda_project_yaml.variables.items() + if not k.startswith("JHUB_APP_CONFIG_") + } + return { + **jhub_apps_variables, + "environment": environment_variables + } + + +def _get_conda_project_config_yaml(directory: str): + """Given the directory, get conda project config object""" + # Moving this to top level import causes this problem: + # https://github.com/jupyter/jupyter_events/issues/99 + from conda_project import CondaProject, CondaProjectError + from conda_project.project_file import CondaProjectYaml + try: + conda_project = CondaProject(directory) + # This is a private attribute, ideally we shouldn't access it, + # but I haven't found an alternative way to get this + conda_project_yaml: CondaProjectYaml = conda_project._project_file + except CondaProjectError as e: + message = "Invalid conda-project" + logger.error(message) + logger.exception(e) + raise HTTPException( + detail=message, + status_code=status.HTTP_400_BAD_REQUEST, + ) + return conda_project_yaml + + +def _check_conda_project_config_directory_exists(repository: Repository, temp_dir: str): + """Check if the conda project config directory provided by the user exists""" + temp_dir_path = pathlib.Path(temp_dir) + conda_project_dir = temp_dir_path / repository.config_directory + if not conda_project_dir.exists(): + message = f"Path '{repository.config_directory}' doesn't exists in the repository." + logger.error(message, repo_url=repository.url) + raise HTTPException( + detail=message, + status_code=status.HTTP_400_BAD_REQUEST, + ) diff --git a/jhub_apps/service/models.py b/jhub_apps/service/models.py index 796b8902..42442b1f 100644 --- a/jhub_apps/service/models.py +++ b/jhub_apps/service/models.py @@ -55,23 +55,34 @@ class HubApiError(BaseModel): detail: HubResponse -class UserOptions(BaseModel): - jhub_app: bool +class Repository(BaseModel): + url: str + config_directory: str = "." + # git ref + ref: str = "main" + + +class JHubAppConfig(BaseModel): display_name: str description: str thumbnail: str = None filepath: typing.Optional[str] = str() framework: str = "panel" custom_command: typing.Optional[str] = str() - conda_env: typing.Optional[str] = str() - # Environment variables - env: typing.Optional[dict] = dict() - profile: typing.Optional[str] = str() # Make app available to public (unauthenticated Hub users) public: typing.Optional[bool] = False # Keep app alive, even when there is no activity # So that it's not killed by idle culler keep_alive: typing.Optional[bool] = False + # Environment variables + env: typing.Optional[dict] = dict() + repository: typing.Optional[Repository] = None + + +class UserOptions(JHubAppConfig): + jhub_app: bool + conda_env: typing.Optional[str] = str() + profile: typing.Optional[str] = str() share_with: typing.Optional[SharePermissions] = None diff --git a/jhub_apps/service/routes.py b/jhub_apps/service/routes.py index 424b828e..8b67e59f 100644 --- a/jhub_apps/service/routes.py +++ b/jhub_apps/service/routes.py @@ -28,6 +28,8 @@ HubApiError, ServerCreation, User, + Repository, + JHubAppConfig, ) from jhub_apps.service.security import get_current_user from jhub_apps.service.utils import ( @@ -37,6 +39,7 @@ get_thumbnail_data_url, get_shared_servers, ) +from jhub_apps.service.app_from_git import _get_app_configuration_from_git from jhub_apps.spawner.types import FRAMEWORKS from jhub_apps.version import get_version @@ -271,6 +274,24 @@ async def hub_services(user: User = Depends(get_current_user)): return hub_client.get_services() +@router.post("/app-config-from-git/",) +async def app_from_git( + repo: Repository, + user: User = Depends(get_current_user) +) -> JHubAppConfig: + """ + ## Fetches jhub-apps application configuration from a git repository. + + Note: This endpoint is kept as POST intentionally because the client is + requesting the server to process some data, in this case, to fetch + a repository, read its conda project config, and return specific values, + which is a processing action. + """ + logger.info("Getting app configuration from git repository") + response = _get_app_configuration_from_git(repo) + return response + + @router.get("/") @router.get("/status") async def status_endpoint(): diff --git a/jhub_apps/spawner/spawner_creation.py b/jhub_apps/spawner/spawner_creation.py index 9d3bc4cd..afa2ddac 100644 --- a/jhub_apps/spawner/spawner_creation.py +++ b/jhub_apps/spawner/spawner_creation.py @@ -1,3 +1,5 @@ +import uuid + import structlog from jhub_apps.spawner.utils import get_origin_host @@ -56,6 +58,18 @@ def get_args(self): command = Command(args=GENERIC_ARGS + custom_cmd.split()) else: command: Command = COMMANDS.get(framework) + + repository = self.user_options.get("repository") + if repository: + logger.info(f"repository specified: {repository}") + # The repository will be cloned during spawn time to + # deploy the app from the repository. + command.args.extend([ + f"--repo={repository.get('url')}", + f"--repofolder=/tmp/{self.name}-{uuid.uuid4().hex[:6]}", + f"--repobranch={repository.get('ref')}" + ]) + command_args = command.get_substituted_args( python_exec=self.config.JAppsConfig.python_exec, filepath=app_filepath, diff --git a/jhub_apps/tests/common/constants.py b/jhub_apps/tests/common/constants.py index af6ae2c2..575afe62 100644 --- a/jhub_apps/tests/common/constants.py +++ b/jhub_apps/tests/common/constants.py @@ -4,3 +4,6 @@ MOCK_USER.name = "jovyan" JUPYTERHUB_HOSTNAME = "127.0.0.1:8000" +JUPYTERHUB_USERNAME = "admin" +JUPYTERHUB_PASSWORD = "admin" +JHUB_APPS_API_BASE_URL = f"http://{JUPYTERHUB_HOSTNAME}/services/japps" diff --git a/jhub_apps/tests/tests_e2e/test_api.py b/jhub_apps/tests/tests_e2e/test_api.py index 37a88daa..fdaa8b27 100644 --- a/jhub_apps/tests/tests_e2e/test_api.py +++ b/jhub_apps/tests/tests_e2e/test_api.py @@ -1,4 +1,110 @@ +import hashlib + +import pytest + +from jhub_apps.service.models import Repository, UserOptions, ServerCreation +from jhub_apps.tests.common.constants import JHUB_APPS_API_BASE_URL, JUPYTERHUB_HOSTNAME +from jhub_apps.tests.tests_e2e.utils import get_jhub_apps_session, fetch_url_until_title_found + +EXAMPLE_TEST_REPO = "https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git" + + def test_api_status(client): response = client.get("/status") assert response.status_code == 200 assert set(response.json().keys()) == {"version", "status"} + + +def test_app_config_from_git_api( + client, +): + response = client.post( + '/app-config-from-git/', + json={ + "url": EXAMPLE_TEST_REPO, + "config_directory": ".", + "ref": "main" + } + ) + assert response.status_code == 200 + response_json = response.json() + assert response_json + assert set(response_json.keys()) == { + "display_name", "description", "framework", "filepath", + "env", "keep_alive", "public", "thumbnail", + "custom_command", "repository" + } + assert response_json["display_name"] == "My Panel App (Git)" + assert response_json["description"] == "This is a panel app created from git repository" + assert response_json["framework"] == "panel" + assert response_json["filepath"] == "panel_basic.py" + assert response_json["env"] == { + "SOMETHING_FOO": "bar", + "SOMETHING_BAR": "beta", + } + assert response_json["keep_alive"] is False + assert response_json["public"] is False + + assert isinstance(response_json["thumbnail"], str) + expected_thumbnail_sha = "a8104b2482360eee525dc696dafcd2a17864687891dc1b6c9e21520518a5ea89" + assert hashlib.sha256(response_json["thumbnail"].encode('utf-8')).hexdigest() == expected_thumbnail_sha + + +@pytest.mark.parametrize("repo_url, config_directory, response_status_code,detail", [ + (EXAMPLE_TEST_REPO, "non-existent-path", 400, + "Path 'non-existent-path' doesn't exists in the repository."), + ("http://invalid-repo/", ".", 400, + "Repository clone failed: http://invalid-repo/"), +]) +def test_app_config_from_git_api_invalid( + client, + repo_url, + config_directory, + response_status_code, + detail +): + response = client.post( + '/app-config-from-git/', + json={ + "url": repo_url, + "config_directory": config_directory, + "ref": "main" + } + ) + assert response.status_code == response_status_code + response_json = response.json() + assert "detail" in response_json + assert response_json["detail"] == detail + + +def test_create_server_with_git_repository(): + user_options = UserOptions( + jhub_app=True, + display_name="Test Application", + description="App description", + framework="panel", + thumbnail="data:image/png;base64,ZHVtbXkgaW1hZ2UgZGF0YQ==", + filepath="panel_basic.py", + repository=Repository( + url="https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git", + ) + ) + files = {"thumbnail": ("test.png", b"dummy image data", "image/png")} + server_data = ServerCreation( + servername="test server from git repo", + user_options=user_options + ) + data = {"data": server_data.model_dump_json()} + session = get_jhub_apps_session() + response = session.post( + f"{JHUB_APPS_API_BASE_URL}/server", + verify=False, + data=data, + files=files + ) + assert response.status_code == 200 + server_name = response.json()[-1] + created_app_url = f"http://{JUPYTERHUB_HOSTNAME}/user/admin/{server_name}/" + fetch_url_until_title_found( + session, url=created_app_url, expected_title="Panel Test App from Git Repository" + ) diff --git a/jhub_apps/tests/tests_e2e/test_integration.py b/jhub_apps/tests/tests_e2e/test_integration.py index 0a11b047..30c970fe 100644 --- a/jhub_apps/tests/tests_e2e/test_integration.py +++ b/jhub_apps/tests/tests_e2e/test_integration.py @@ -148,8 +148,6 @@ def sign_in_and_authorize(page, username, password): page.get_by_label("Password:").fill(password) logger.info("Pressing Sign in button") page.get_by_role("button", name="Sign in").click() - logger.info("Click Authorize button") - page.get_by_role("button", name="Authorize").click() def sign_out(page): diff --git a/jhub_apps/tests/tests_e2e/utils.py b/jhub_apps/tests/tests_e2e/utils.py new file mode 100644 index 00000000..3ca4225d --- /dev/null +++ b/jhub_apps/tests/tests_e2e/utils.py @@ -0,0 +1,60 @@ +import time + +import requests + +from jhub_apps.tests.common.constants import JUPYTERHUB_HOSTNAME, JUPYTERHUB_USERNAME, JUPYTERHUB_PASSWORD + + +def get_jhub_apps_session(): + """Get jhub-apps session with authenticated cookies to be able to call jhub-apps API""" + session = requests.Session() + session.cookies.clear() + try: + response = session.get( + f"http://{JUPYTERHUB_HOSTNAME}/hub/login", verify=False + ) + response.raise_for_status() + auth_data = { + "_xsrf": session.cookies['_xsrf'], + "username": JUPYTERHUB_USERNAME, + "password": JUPYTERHUB_PASSWORD, + } + response = session.post( + f"http://{JUPYTERHUB_HOSTNAME}/hub/login?next=", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data=auth_data, + verify=False, + ) + response.raise_for_status() + + except requests.RequestException as e: + raise ValueError(f"An error occurred during authentication: {e}") + + response_login = session.get( + f"http://{JUPYTERHUB_HOSTNAME}/services/japps/jhub-login", + ) + response_login.raise_for_status() + response_user = session.get( + f"http://{JUPYTERHUB_HOSTNAME}/services/japps/user", + verify=False + ) + response_user.raise_for_status() + return session + + +def fetch_url_until_title_found( + session, url, expected_title, timeout=10, interval=2 +): + """Fetches url until the expected title is found.""" + start_time = time.time() + while True: + try: + response = session.get(url) + assert response.status_code == 200 + if expected_title in str(response.content): + return + except (requests.RequestException, ValueError, AssertionError) as e: + time_elapsed = time.time() - start_time + if time_elapsed > timeout: + raise TimeoutError(f"Failed to get the title {expected_title} within {timeout} seconds") from e + time.sleep(interval) diff --git a/jhub_apps/tests/tests_unit/test_api.py b/jhub_apps/tests/tests_unit/test_api.py index 277a8f96..eea18882 100644 --- a/jhub_apps/tests/tests_unit/test_api.py +++ b/jhub_apps/tests/tests_unit/test_api.py @@ -5,6 +5,7 @@ import pytest from jhub_apps.hub_client.hub_client import HubClient +from jhub_apps.service.models import UserOptions, ServerCreation, Repository from jhub_apps.service.utils import get_shared_servers from jhub_apps.spawner.types import FRAMEWORKS from jhub_apps.tests.common.constants import MOCK_USER @@ -224,3 +225,32 @@ def test_open_api_docs(client): assert response.status_code == 200 rjson = response.json() assert rjson['info']['version'] + + +@patch.object(HubClient, "create_server") +def test_create_server_with_git_repository( + hub_create_server, + client, +): + user_options = UserOptions( + jhub_app=True, + display_name="Test Application", + description="App description", + framework="panel", + thumbnail="data:image/png;base64,ZHVtbXkgaW1hZ2UgZGF0YQ==", + filepath="panel_basic.py", + repository=Repository( + url="https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git", + ) + ) + server_data = ServerCreation(servername="test server", user_options=user_options) + files = {"thumbnail": ("test.png", b"dummy image data", "image/png")} + data = {"data": server_data.model_dump_json()} + hub_create_server.return_value = (201, 'test-server-abcdef') + response = client.post("/server", data=data, files=files) + assert response.status_code == 200 + assert response.json() == [201, 'test-server-abcdef'] + hub_create_server.assert_called_once_with( + username="jovyan", servername=server_data.servername, + user_options=user_options + ) diff --git a/jhub_apps/tests/tests_unit/test_app_from_git.py b/jhub_apps/tests/tests_unit/test_app_from_git.py new file mode 100644 index 00000000..747dcc3b --- /dev/null +++ b/jhub_apps/tests/tests_unit/test_app_from_git.py @@ -0,0 +1,31 @@ +from unittest.mock import Mock + +from jhub_apps.service.app_from_git import _extract_jhub_apps_config_from_conda_project_config + + +def test_extract_jhub_apps_config_from_conda_project_config(): + conda_project_yaml = Mock(variables={ + "JHUB_APP_CONFIG_name": "My Panel App (Git)", + "JHUB_APP_CONFIG_description": "This is a panel app created from git repository", + "JHUB_APP_CONFIG_framework": "panel", + "JHUB_APP_CONFIG_filepath": "panel_basic.py", + "JHUB_APP_CONFIG_keep_alive": "false", + "JHUB_APP_CONFIG_public": "false", + "JHUB_APP_CONFIG_thumbnail_path": "panel.png", + "SOMETHING_FOO": "bar", + "SOMETHING_BAR": "beta", + }) + jhub_apps_config = _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml) + assert jhub_apps_config == { + "name": "My Panel App (Git)", + "description": "This is a panel app created from git repository", + "framework": "panel", + "filepath": "panel_basic.py", + "keep_alive": "false", + "public": "false", + "thumbnail_path": "panel.png", + "environment": { + "SOMETHING_FOO": "bar", + "SOMETHING_BAR": "beta", + } + } diff --git a/jupyterhub_config.py b/jupyterhub_config.py index 84cbcc7e..8dcf3f44 100644 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -19,7 +19,7 @@ c.JAppsConfig.service_workers = 1 c.JupyterHub.default_url = "/hub/home" -c = install_jhub_apps(c, spawner_to_subclass=SimpleLocalProcessSpawner) +c = install_jhub_apps(c, spawner_to_subclass=SimpleLocalProcessSpawner, oauth_no_confirm=True) c.JupyterHub.template_paths = theme_template_paths diff --git a/pyproject.toml b/pyproject.toml index 9df7e10d..5066198a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,10 @@ dependencies = [ "cachetools", "structlog", "PyJWT", + "GitPython", + # pinning to avoid unexpected changes in spec causing + # unexpected breakage + "conda-project==0.4.2" ] dynamic = ["version"]