Deploy app from git repository (#445)

* get app config from git

* add documentation for getting app config from git

* remove unused vars

* add git url in spawner

* add example

* add positive and neg tests

* move app from repo test to e2e section

* add test for invalid repository

* improve description

* git repo branch add to user options

* add test for create server from git repo

* replace aktech with jovyan

* use conda-project instead

* fix linting

* move conda project import to inside function

* remove unnecessary config file

* app config from conda project

* log message on failure

* move app from git to separate module

* move app from git to a separate directory

* add more docs

* add test to parse jhub-app config from conda-project yml

* use repository object for server creation

* update create server with git repo test

* add integration tests for api

* set oauth_no_confirm=True

* remove clicking on authorize

* always create artifact name

* extract common items for user options

* add a line about cloning repository

* add another suffix to app name

* Apply suggestions from code review

Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com>

---------

Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com>

.github/workflows/test-integration.yml

@@ -92,6 +92,7 @@ jobs:
 
     - name: Create artifact name
       id: artifact-name
+      if: always()
       run: |
         if [ "${{ matrix.jupyterhub }}" = "4.1.5" ]; then
             jhub_suffix="4x"

environment-dev.yml

@@ -22,3 +22,5 @@ dependencies:
   - cachetools
   - structlog
   - gradio
+  - gitpython
+  - conda-project=0.4.2

jhub_apps/service/app_from_git.py

@@ -0,0 +1,125 @@
+import os
+import pathlib
+import tempfile
+from pathlib import Path
+
+import git
+from fastapi import HTTPException, status
+from pydantic import ValidationError
+
+from jhub_apps.service.models import Repository, JHubAppConfig
+from jhub_apps.service.utils import logger, encode_file_to_data_url
+
+
+def _clone_repo(repository: Repository, temp_dir):
+    """Clone repository to the given tem_dir"""
+    try:
+        logger.info("Trying to clone repository", repo_url=repository.url)
+        git.Repo.clone_from(repository.url, temp_dir, depth=1, branch=repository.ref)
+    except Exception as e:
+        message = f"Repository clone failed: {repository.url}"
+        logger.error(message, repo_url=repository.url)
+        logger.error(e)
+        raise HTTPException(
+            detail=message,
+            status_code=status.HTTP_400_BAD_REQUEST,
+        )
+
+
+def _get_app_configuration_from_git(
+        repository: Repository
+) -> JHubAppConfig:
+    """Clones the git directory into a temporary path and extracts all the metadata
+    about the app from conda-project's config yaml.
+    """
+    with tempfile.TemporaryDirectory() as temp_dir:
+        _clone_repo(repository, temp_dir)
+        _check_conda_project_config_directory_exists(repository, temp_dir)
+        conda_project_yaml = _get_conda_project_config_yaml(temp_dir)
+        jhub_apps_config_dict = _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml)
+        app_config = _load_jhub_app_config_to_pydantic_model(
+            jhub_apps_config_dict,
+            repository,
+            temp_dir
+        )
+        return app_config
+
+
+def _load_jhub_app_config_to_pydantic_model(
+        jhub_apps_config_dict: dict, repository: Repository, temp_dir: str
+):
+    """Load the parsed jhub-apps config into pydantic model for validation"""
+    thumbnail_base64 = ""
+    thumbnail_path_from_config = jhub_apps_config_dict.get("thumbnail_path")
+    if thumbnail_path_from_config:
+        thumbnail_path = Path(os.path.join(temp_dir, thumbnail_path_from_config))
+        thumbnail_base64 = encode_file_to_data_url(
+            filename=thumbnail_path.name, file_contents=thumbnail_path.read_bytes()
+        )
+    try:
+        # Load YAML content into the Pydantic model
+        app_config = JHubAppConfig(**{
+            **jhub_apps_config_dict,
+            "repository": repository,
+            "thumbnail": thumbnail_base64,
+            "env": jhub_apps_config_dict.get("environment", {})
+        })
+    except ValidationError as e:
+        message = f"Validation error: {e}"
+        logger.error(message)
+        raise HTTPException(
+            detail=message,
+            status_code=status.HTTP_400_BAD_REQUEST,
+        )
+    return app_config
+
+
+def _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml):
+    """Extracts jhub-apps app config from conda project yaml's config"""
+    jhub_apps_variables = {
+        k.split("JHUB_APP_CONFIG_")[-1]: v for k, v in conda_project_yaml.variables.items()
+        if k.startswith("JHUB_APP_CONFIG_")
+    }
+    environment_variables = {
+        k: v for k, v in conda_project_yaml.variables.items()
+        if not k.startswith("JHUB_APP_CONFIG_")
+    }
+    return {
+        **jhub_apps_variables,
+        "environment": environment_variables
+    }
+
+
+def _get_conda_project_config_yaml(directory: str):
+    """Given the directory, get conda project config object"""
+    # Moving this to top level import causes this problem:
+    # https://github.com/jupyter/jupyter_events/issues/99
+    from conda_project import CondaProject, CondaProjectError
+    from conda_project.project_file import CondaProjectYaml
+    try:
+        conda_project = CondaProject(directory)
+        # This is a private attribute, ideally we shouldn't access it,
+        # but I haven't found an alternative way to get this
+        conda_project_yaml: CondaProjectYaml = conda_project._project_file
+    except CondaProjectError as e:
+        message = "Invalid conda-project"
+        logger.error(message)
+        logger.exception(e)
+        raise HTTPException(
+            detail=message,
+            status_code=status.HTTP_400_BAD_REQUEST,
+        )
+    return conda_project_yaml
+
+
+def _check_conda_project_config_directory_exists(repository: Repository, temp_dir: str):
+    """Check if the conda project config directory provided by the user exists"""
+    temp_dir_path = pathlib.Path(temp_dir)
+    conda_project_dir = temp_dir_path / repository.config_directory
+    if not conda_project_dir.exists():
+        message = f"Path '{repository.config_directory}' doesn't exists in the repository."
+        logger.error(message, repo_url=repository.url)
+        raise HTTPException(
+            detail=message,
+            status_code=status.HTTP_400_BAD_REQUEST,
+        )

jhub_apps/service/models.py

@@ -55,23 +55,34 @@ class HubApiError(BaseModel):
     detail: HubResponse
 
 
-class UserOptions(BaseModel):
-    jhub_app: bool
+class Repository(BaseModel):
+    url: str
+    config_directory: str = "."
+    # git ref
+    ref: str = "main"
+
+
+class JHubAppConfig(BaseModel):
     display_name: str
     description: str
     thumbnail: str = None
     filepath: typing.Optional[str] = str()
     framework: str = "panel"
     custom_command: typing.Optional[str] = str()
-    conda_env: typing.Optional[str] = str()
-    # Environment variables
-    env: typing.Optional[dict] = dict()
-    profile: typing.Optional[str] = str()
     # Make app available to public (unauthenticated Hub users)
     public: typing.Optional[bool] = False
     # Keep app alive, even when there is no activity
     # So that it's not killed by idle culler
     keep_alive: typing.Optional[bool] = False
+    # Environment variables
+    env: typing.Optional[dict] = dict()
+    repository: typing.Optional[Repository] = None
+
+
+class UserOptions(JHubAppConfig):
+    jhub_app: bool
+    conda_env: typing.Optional[str] = str()
+    profile: typing.Optional[str] = str()
     share_with: typing.Optional[SharePermissions] = None
 
 

jhub_apps/service/routes.py

@@ -28,6 +28,8 @@
     HubApiError,
     ServerCreation,
     User,
+    Repository,
+    JHubAppConfig,
 )
 from jhub_apps.service.security import get_current_user
 from jhub_apps.service.utils import (
@@ -37,6 +39,7 @@
     get_thumbnail_data_url,
     get_shared_servers,
 )
+from jhub_apps.service.app_from_git import _get_app_configuration_from_git
 from jhub_apps.spawner.types import FRAMEWORKS
 from jhub_apps.version import get_version
 
@@ -271,6 +274,24 @@ async def hub_services(user: User = Depends(get_current_user)):
     return hub_client.get_services()
 
 
+@router.post("/app-config-from-git/",)
+async def app_from_git(
+        repo: Repository,
+        user: User = Depends(get_current_user)
+) -> JHubAppConfig:
+    """
+    ## Fetches jhub-apps application configuration from a git repository.
+
+    Note: This endpoint is kept as POST intentionally because the client is
+    requesting the server to process some data, in this case, to fetch
+    a repository, read its conda project config, and return specific values,
+    which is a processing action.
+    """
+    logger.info("Getting app configuration from git repository")
+    response = _get_app_configuration_from_git(repo)
+    return response
+
+
 @router.get("/")
 @router.get("/status")
 async def status_endpoint():

jhub_apps/spawner/spawner_creation.py

@@ -1,3 +1,5 @@
+import uuid
+
 import structlog
 
 from jhub_apps.spawner.utils import get_origin_host
@@ -56,6 +58,18 @@ def get_args(self):
                     command = Command(args=GENERIC_ARGS + custom_cmd.split())
                 else:
                     command: Command = COMMANDS.get(framework)
+
+                repository = self.user_options.get("repository")
+                if repository:
+                    logger.info(f"repository specified: {repository}")
+                    # The repository will be cloned during spawn time to
+                    # deploy the app from the repository.
+                    command.args.extend([
+                        f"--repo={repository.get('url')}",
+                        f"--repofolder=/tmp/{self.name}-{uuid.uuid4().hex[:6]}",
+                        f"--repobranch={repository.get('ref')}"
+                    ])
+
                 command_args = command.get_substituted_args(
                     python_exec=self.config.JAppsConfig.python_exec,
                     filepath=app_filepath,

jhub_apps/tests/common/constants.py

@@ -4,3 +4,6 @@
 MOCK_USER.name = "jovyan"
 
 JUPYTERHUB_HOSTNAME = "127.0.0.1:8000"
+JUPYTERHUB_USERNAME = "admin"
+JUPYTERHUB_PASSWORD = "admin"
+JHUB_APPS_API_BASE_URL = f"http://{JUPYTERHUB_HOSTNAME}/services/japps"

jhub_apps/tests/tests_e2e/test_api.py

@@ -1,4 +1,110 @@
+import hashlib
+
+import pytest
+
+from jhub_apps.service.models import Repository, UserOptions, ServerCreation
+from jhub_apps.tests.common.constants import JHUB_APPS_API_BASE_URL, JUPYTERHUB_HOSTNAME
+from jhub_apps.tests.tests_e2e.utils import get_jhub_apps_session, fetch_url_until_title_found
+
+EXAMPLE_TEST_REPO = "https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git"
+
+
 def test_api_status(client):
     response = client.get("/status")
     assert response.status_code == 200
     assert set(response.json().keys()) == {"version", "status"}
+
+
+def test_app_config_from_git_api(
+        client,
+):
+    response = client.post(
+        '/app-config-from-git/',
+        json={
+            "url": EXAMPLE_TEST_REPO,
+            "config_directory": ".",
+            "ref": "main"
+        }
+    )
+    assert response.status_code == 200
+    response_json = response.json()
+    assert response_json
+    assert set(response_json.keys()) == {
+        "display_name", "description", "framework", "filepath",
+        "env", "keep_alive", "public", "thumbnail",
+        "custom_command", "repository"
+    }
+    assert response_json["display_name"] == "My Panel App (Git)"
+    assert response_json["description"] == "This is a panel app created from git repository"
+    assert response_json["framework"] == "panel"
+    assert response_json["filepath"] == "panel_basic.py"
+    assert response_json["env"] == {
+        "SOMETHING_FOO": "bar",
+        "SOMETHING_BAR": "beta",
+    }
+    assert response_json["keep_alive"] is False
+    assert response_json["public"] is False
+
+    assert isinstance(response_json["thumbnail"], str)
+    expected_thumbnail_sha = "a8104b2482360eee525dc696dafcd2a17864687891dc1b6c9e21520518a5ea89"
+    assert hashlib.sha256(response_json["thumbnail"].encode('utf-8')).hexdigest() == expected_thumbnail_sha
+
+
+@pytest.mark.parametrize("repo_url, config_directory, response_status_code,detail", [
+    (EXAMPLE_TEST_REPO, "non-existent-path", 400,
+     "Path 'non-existent-path' doesn't exists in the repository."),
+    ("http://invalid-repo/", ".", 400,
+     "Repository clone failed: http://invalid-repo/"),
+])
+def test_app_config_from_git_api_invalid(
+        client,
+        repo_url,
+        config_directory,
+        response_status_code,
+        detail
+):
+    response = client.post(
+        '/app-config-from-git/',
+        json={
+            "url": repo_url,
+            "config_directory": config_directory,
+            "ref": "main"
+        }
+    )
+    assert response.status_code == response_status_code
+    response_json = response.json()
+    assert "detail" in response_json
+    assert response_json["detail"] == detail
+
+
+def test_create_server_with_git_repository():
+    user_options = UserOptions(
+        jhub_app=True,
+        display_name="Test Application",
+        description="App description",
+        framework="panel",
+        thumbnail="data:image/png;base64,ZHVtbXkgaW1hZ2UgZGF0YQ==",
+        filepath="panel_basic.py",
+        repository=Repository(
+            url="https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git",
+        )
+    )
+    files = {"thumbnail": ("test.png", b"dummy image data", "image/png")}
+    server_data = ServerCreation(
+        servername="test server from git repo",
+        user_options=user_options
+    )
+    data = {"data": server_data.model_dump_json()}
+    session = get_jhub_apps_session()
+    response = session.post(
+        f"{JHUB_APPS_API_BASE_URL}/server",
+        verify=False,
+        data=data,
+        files=files
+    )
+    assert response.status_code == 200
+    server_name = response.json()[-1]
+    created_app_url = f"http://{JUPYTERHUB_HOSTNAME}/user/admin/{server_name}/"
+    fetch_url_until_title_found(
+        session, url=created_app_url, expected_title="Panel Test App from Git Repository"
+    )

jhub_apps/tests/tests_e2e/test_integration.py

@@ -148,8 +148,6 @@ def sign_in_and_authorize(page, username, password):
     page.get_by_label("Password:").fill(password)
     logger.info("Pressing Sign in button")
     page.get_by_role("button", name="Sign in").click()
-    logger.info("Click Authorize button")
-    page.get_by_role("button", name="Authorize").click()
 
 
 def sign_out(page):