diff --git a/.gitignore b/.gitignore index 09448f7b7..ea073886f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ data lifemonitor/static/dist lifemonitor/static/src/node_modules docker-compose.yml +.*docker-compose.yml utils/certs/data tests/config/data/crates/*.zip tests/config/registries/seek/data diff --git a/.lifemonitor.yml b/.lifemonitor.yml new file mode 100644 index 000000000..4b1d6c4d8 --- /dev/null +++ b/.lifemonitor.yml @@ -0,0 +1,44 @@ +# worfklow name (override name defined on the RO-Crate metadata) +# name: MyWorkflow +# worfklow visibility +public: False + +# Issue Checker Settings +issues: + # Enable/Disable issue checker + # The list of issue types can be found @ /workflows/issues + # (e.g., https://api.lifemonitor.eu/workflows/issues) + check: true + # csv of issues to check (all included by default) + # include: [missing_config_file, not_initialised_repository_issue, missing_workflow_file, missing_metadata_file, missing_ro_crate_workflow_file, outdated_metadata_file, missing_workflow_name] + # csv of issues to ignore (none ignored by default) + # exclude: [missing_config_file, not_initialised_repository_issue, missing_workflow_file, missing_metadata_file, missing_ro_crate_workflow_file, outdated_metadata_file, missing_workflow_name] + + +# Github Integration Settings +push: + branches: + # Define the list of branches to watch + # - name: feature/XXX # wildcards can be used to specify branches (e.g., feature/*) + # update_registries: ["wfhubdev"] # available registries are listed + # # by the endpoint `/registries` + # # (e.g., https://api.lifemontor.eu/registries) + - name: "main" + update_registries: [] + enable_notifications: true + # - name: "develop" + # update_registries: [] + # enable_notifications: true + + tags: + # Define the list of tags to watch + # - name: v*.*.* # wildcards can be used to specify tags (e.g., feature/*) + # update_registries: ["wfhub"] # available registries are listed + # # by the endpoint `/registries` + # # (e.g., https://api.lifemontor.eu/registries) + - name: "v*.*.*" + update_registries: [wfhubdev, wfhubprod] + enable_notifications: true + - name: "*.*.*" + update_registries: [wfhubdev, wfhubprod] + enable_notifications: true \ No newline at end of file diff --git a/Makefile b/Makefile index 7b5f7ac17..cb8bae901 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,13 @@ define get_opts $(shell opts=""; values=($(2)); for (( i=0; i<$${#values[@]}; i++)); do opts="$$opts --$(1) '$${values[$$i]}'"; done; echo "$$opts") endef +# set docker-compose command +ifeq ($(shell command -v "docker-compose" 2> /dev/null),) + docker_compose := docker compose +else + docker_compose := docker-compose +endif + # default Docker build options build_kit := build_cmd := build @@ -78,7 +85,6 @@ ifdef PLATFORMS endif - all: images images: lifemonitor smeeio @@ -88,9 +94,15 @@ compose-files: docker-compose.base.yml \ docker-compose.dev.yml \ docker-compose.extra.yml \ docker-compose.test.yml \ - docker-compose.prom.yml \ + docker-compose.monitoring.yml \ settings.conf +prod: + $(eval LM_MODE=prod) + +dev: + $(eval LM_MODE=dev) + certs: @# Generate certificates if they do not exist \ if ! [[ -f "certs/lm.key" && -f "certs/lm.key" && -f "certs/lifemonitor.ca.crt" ]]; then \ @@ -160,41 +172,46 @@ aux_images: tests/config/registries/seek/seek.Dockerfile certs tests/config/registries/seek/ ; \ printf "$(done)\n" -start: images compose-files ## Start LifeMonitor in a Production environment +start: images compose-files prod reset_compose ## Start LifeMonitor in a Production environment @printf "\n$(bold)Starting production services...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose $${base} \ + $(docker_compose) $${base} \ -f docker-compose.prod.yml \ -f docker-compose.base.yml \ - -f docker-compose.prom.yml \ + -f docker-compose.monitoring.yml \ config)" > docker-compose.yml \ - && docker-compose -f docker-compose.yml up -d redis db init lm worker nginx prometheus;\ + && cp {,.prod.}docker-compose.yml \ + && $(docker_compose) -f docker-compose.yml up -d redis db init lm worker ws_server nginx prometheus ;\ printf "$(done)\n" -start-dev: images compose-files ## Start LifeMonitor in a Development environment +start-dev: images compose-files dev reset_compose ## Start LifeMonitor in a Development environment @printf "\n$(bold)Starting development services...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose $${base} \ + $(docker_compose) $${base} \ -f docker-compose.base.yml \ + -f docker-compose.monitoring.yml \ -f docker-compose.dev.yml \ config)" > docker-compose.yml \ - && docker-compose -f docker-compose.yml up -d redis db dev_proxy github_event_proxy init lm worker ;\ + && cp {,.dev.}docker-compose.yml \ + && $(docker_compose) -f docker-compose.yml up -d redis db dev_proxy github_event_proxy init lm worker ws_server prometheus ;\ printf "$(done)\n" -start-testing: compose-files aux_images ro_crates images ## Start LifeMonitor in a Testing environment +start-testing: compose-files aux_images ro_crates images reset_compose ## Start LifeMonitor in a Testing environment @printf "\n$(bold)Starting testing services...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose $${base} \ + $(docker_compose) $${base} \ -f docker-compose.extra.yml \ -f docker-compose.base.yml \ + -f docker-compose.monitoring.yml \ -f docker-compose.dev.yml \ -f docker-compose.test.yml \ config)" > docker-compose.yml \ - && docker-compose -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ;\ - docker-compose -f ./docker-compose.yml \ + && cp {,.test.}docker-compose.yml \ + && $(docker_compose) -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ws_server ;\ + $(docker_compose) -f ./docker-compose.yml \ exec -T lmtests /bin/bash -c "tests/wait-for-it.sh seek:3000 -t 600"; \ printf "$(done)\n" @@ -202,43 +219,44 @@ start-nginx: certs docker-compose.prod.yml ## Start a nginx front-end proxy for @printf "\n$(bold)Starting nginx proxy...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose $${base} \ + $(docker_compose) $${base} \ -f docker-compose.prod.yml \ -f docker-compose.base.yml config)" > docker-compose.yml \ - && docker-compose up -d nginx ; \ + && $(docker_compose) up -d nginx ; \ printf "$(done)\n" start-aux-services: aux_images ro_crates docker-compose.extra.yml ## Start auxiliary services (i.e., Jenkins, Seek) useful for development and testing @printf "\n$(bold)Starting auxiliary services...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose $${base} -f docker-compose.extra.yml config)" > docker-compose.yml \ - && docker-compose up -d seek jenkins ; \ + $(docker_compose) $${base} -f docker-compose.extra.yml config)" > docker-compose.yml \ + && $(docker_compose) up -d seek jenkins ; \ printf "$(done)\n" # start-jupyter: aux_images docker-compose.extra.yml ## Start jupyter service # @printf "\n$(bold)Starting jupyter service...$(reset)\n" ; \ # base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ # echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ -# docker-compose $${base} -f docker-compose.jupyter.yml config)" > docker-compose.yml \ -# && docker-compose up -d jupyter ; \ +# $(docker_compose) $${base} -f docker-compose.jupyter.yml config)" > docker-compose.yml \ +# && $(docker_compose) up -d jupyter ; \ # printf "$(done)\n" run-tests: start-testing ## Run all tests in the Testing Environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose exec -T lmtests /bin/bash -c "pytest --color=yes tests" + $(docker_compose) exec -T lmtests /bin/bash -c "pytest --color=yes tests" tests: start-testing ## CI utility to setup, run tests and teardown a testing environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ - docker-compose -f ./docker-compose.yml \ + $(docker_compose) -f ./docker-compose.yml \ exec -T lmtests /bin/bash -c "pytest --color=yes tests"; \ result=$$?; \ printf "\n$(bold)Teardown services...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose -f docker-compose.extra.yml \ + $(docker_compose) -f docker-compose.extra.yml \ -f docker-compose.base.yml \ + -f docker-compose.monitoring.yml \ -f docker-compose.dev.yml \ -f docker-compose.test.yml \ down ; \ @@ -247,61 +265,75 @@ tests: start-testing ## CI utility to setup, run tests and teardown a testing en stop-aux-services: docker-compose.extra.yml ## Stop all auxiliary services (i.e., Jenkins, Seek) @echo "$(bold)Teardown auxiliary services...$(reset)" ; \ - docker-compose -f docker-compose.extra.yml --log-level ERROR stop ; \ + $(docker_compose) -f docker-compose.extra.yml --log-level ERROR stop ; \ printf "$(done)\n" # stop-jupyter: docker-compose.jupyter.yml ## Stop jupyter service # @echo "$(bold)Stopping auxiliary services...$(reset)" ; \ -# docker-compose -f docker-compose.jupyter.yml --log-level ERROR stop ; \ +# $(docker_compose) -f docker-compose.jupyter.yml --log-level ERROR stop ; \ # printf "$(done)\n" stop-nginx: docker-compose.yml ## Stop the nginx front-end proxy for the LifeMonitor back-end @echo "$(bold)Stopping nginx service...$(reset)" ; \ - docker-compose -f docker-compose.yml stop nginx ; \ + $(docker_compose) -f docker-compose.yml stop nginx ; \ printf "$(done)\n" stop-testing: compose-files ## Stop all the services in the Testing Environment @echo "$(bold)Stopping services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose -f docker-compose.extra.yml \ + $(docker_compose) -f docker-compose.extra.yml \ -f docker-compose.base.yml \ -f docker-compose.dev.yml \ -f docker-compose.test.yml \ - --log-level ERROR stop db lmtests seek jenkins webserver worker ; \ + --log-level ERROR stop db lmtests seek jenkins webserver worker ws_server ; \ printf "$(done)\n" stop-dev: compose-files ## Stop all services in the Develop Environment @echo "$(bold)Stopping development services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose -f docker-compose.base.yml \ + $(docker_compose) -f docker-compose.base.yml \ -f docker-compose.dev.yml \ - stop init lm db github_event_proxy dev_proxy redis worker; \ + stop init lm db github_event_proxy dev_proxy redis worker ws_server prometheus ; \ printf "$(done)\n" stop: compose-files ## Stop all the services in the Production Environment @echo "$(bold)Stopping production services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose -f docker-compose.base.yml \ + $(docker_compose) -f docker-compose.base.yml \ -f docker-compose.prod.yml \ - -f docker-compose.prom.yml \ - --log-level ERROR stop init nginx lm db prometheus redis worker; \ + -f docker-compose.monitoring.yml \ + --log-level ERROR stop init nginx lm db prometheus redis worker ws_server ; \ printf "$(done)\n" stop-all: ## Stop all the services @if [[ -f "docker-compose.yml" ]]; then \ echo "$(bold)Stopping all services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose stop ; \ + $(docker_compose) stop ; \ printf "$(done)\n" ; \ else \ printf "\n$(yellow)WARNING: nothing to remove. 'docker-compose.yml' file not found!$(reset)\n\n" ; \ fi +reset_compose: + @if [[ -f "docker-compose.yml" ]]; then \ + cmp -s docker-compose.yml .$(LM_MODE).docker-compose.yml ; \ + RETVAL=$$? ; \ + if [ $${RETVAL} -ne 0 ]; then \ + current_mode=$$(if [[ "${LM_MODE}" == "DEV" ]]; then echo "production" ; else echo "development" ; fi) ; \ + echo "$(bold)Teardown $${current_mode} services...$(reset)" ; \ + USER_UID=$$(id -u) USER_GID=$$(id -g) \ + $(docker_compose) down ; \ + rm docker-compose.yml ; \ + printf "$(done)\n" ; \ + fi \ + fi + down: ## Teardown all the services @if [[ -f "docker-compose.yml" ]]; then \ echo "$(bold)Teardown all services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose down ; \ + $(docker_compose) down ; \ printf "$(done)\n" ; \ else \ printf "\n$(yellow)WARNING: nothing to remove. 'docker-compose.yml' file not found!$(reset)\n\n" ; \ @@ -311,7 +343,7 @@ clean: ## Clean up the working environment (i.e., running services, network, vol @if [[ -f "docker-compose.yml" ]]; then \ echo "$(bold)Teardown all services...$(reset)" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - docker-compose down -v --remove-orphans ; \ + $(docker_compose) down -v --remove-orphans ; \ printf "$(done)\n"; \ else \ printf "$(yellow)WARNING: nothing to remove. 'docker-compose.yml' file not found!$(reset)\n" ; \ @@ -321,7 +353,7 @@ clean: ## Clean up the working environment (i.e., running services, network, vol rm -rf utils/certs/data @printf "$(done)\n" @printf "\n$(bold)Removing temp files...$(reset) " ; \ - rm -rf docker-compose.yml + rm -rf {,.prod.,.dev.,.test.}docker-compose.yml @printf "$(done)\n\n" .DEFAULT_GOAL := help @@ -329,8 +361,9 @@ clean: ## Clean up the working environment (i.e., running services, network, vol help: ## Show help @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) -.PHONY: all images aux_images certs lifemonitor smeeio ro_crates webserver \ +.PHONY: all images aux_images certs prod dev \ + lifemonitor smeeio ro_crates webserver \ start start-dev start-testing start-nginx start-aux-services \ run-tests tests \ stop-aux-services stop-nginx stop-testing \ - stop-dev stop stop-all down clean + stop-dev stop stop-all down reset_compose clean diff --git a/app.py b/app.py index df1a85e9e..8e5dfe17c 100644 --- a/app.py +++ b/app.py @@ -18,18 +18,27 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import logging import os import ssl from lifemonitor.app import create_app +# initialise logger +logger = logging.getLogger(__name__) + # create an app instance -application = create_app() +application = create_app(init_app=True) -if __name__ == '__main__': - """ Start development server""" + +def start_app_server(): + """ Start Flask App""" context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) context.load_cert_chain( os.environ.get("LIFEMONITOR_TLS_CERT", './certs/lm.crt'), os.environ.get("LIFEMONITOR_TLS_KEY", './certs/lm.key')) application.run(host="0.0.0.0", port=8000, ssl_context=context) + + +if __name__ == '__main__': + start_app_server() diff --git a/cli/client/issues.py b/cli/client/issues.py index 3e0ca3d33..4097a51ac 100644 --- a/cli/client/issues.py +++ b/cli/client/issues.py @@ -24,10 +24,12 @@ import sys import click +import networkx as nx from cli.client.utils import get_repository, init_output_path -from flask.cli import with_appcontext +# from flask.cli import with_appcontext from lifemonitor.api.models.issues import (WorkflowRepositoryIssue, - find_issue_types, load_issue) + find_issue_types, load_issue, + get_issue_graph, ROOT_ISSUE) from lifemonitor.utils import to_snake_case from rich.console import Console from rich.panel import Panel @@ -35,16 +37,24 @@ from rich.syntax import Syntax from rich.table import Table from rich.text import Text +from rich.theme import Theme logger = logging.getLogger(__name__) issues_list = find_issue_types() -console = Console() +custom_theme = Theme({ + "info": "dim cyan", + "warning": "magenta", + "error": "bold red" +}) + + +console = Console(theme=custom_theme) error_console = Console(stderr=True, style="bold red") repository_arg = click.argument('repository', type=str, default=".") -output_path_arg = click.option('-o', '--output-path', type=click.Path(file_okay=False), default=None) +output_path_arg = click.argument('output-path', type=click.Path(file_okay=False)) @click.group(name="issues", help="Tools to develop and check issue types") @@ -99,12 +109,12 @@ def get(config, issue_number): @repository_arg @output_path_arg @click.pass_obj -@with_appcontext -def check(config, repository, output_path=None): +# @with_appcontext +def check(config, repository, output_path): try: init_output_path(output_path=output_path) repo = get_repository(repository, local_path=output_path) - result = repo.check(repository) + result = repo.check(fail_fast=False) # Configure Table table = Table(title=f"Check Issue Report of Repo [bold]{repository}[/bold]", style="bold", expand=True) @@ -115,7 +125,10 @@ def check(config, repository, output_path=None): table.add_column("Tags", style="cyan", overflow="fold", justify="center") checked = [_.name for _ in result.checked] issues = [_.name for _ in result.issues] - for issue in issues_list: + issue_graph = get_issue_graph() + for issue in nx.traversal.bfs_tree(issue_graph, ROOT_ISSUE): + if issue == ROOT_ISSUE: + continue x = None if issue.name not in checked: status = Text("Skipped", style="yellow bold") @@ -127,7 +140,12 @@ def check(config, repository, output_path=None): table.add_row(issue.get_identifier(), issue.name, status, ", ".join([_.path for _ in x.get_changes(repository)]) if x else "", ", ".join(issue.labels)) console.print(table) - + # show optional messages + console.print("\n\n") + for issue in result.issues: + for message in issue._messages: + console.print(f"[{message.type.value}]{message.type.name}:[/{message.type.value}] {message.text}") + console.print("\n\n") except Exception as e: logger.exception(e) error_console.print(str(e)) @@ -140,7 +158,7 @@ def check(config, repository, output_path=None): @repository_arg @output_path_arg @click.pass_obj -@with_appcontext +# @with_appcontext def test(config, issue_file, issue_class, write, repository, output_path=None): proposed_files = [] try: @@ -148,10 +166,10 @@ def test(config, issue_file, issue_class, write, repository, output_path=None): init_output_path(output_path=output_path) logger.debug(issue_file) repo = get_repository(repository, local_path=output_path) - file_issues = load_issue(issue_file) - logger.debug("File issues: %r", [_.name for _ in file_issues]) - issues_list = [_() for _ in file_issues if not issue_class or _.__name__ in issue_class] - logger.debug("Issue: %r", issues_list) + issues_types = load_issue(issue_file) + logger.debug("Types of issues: %r", [_ for _ in issues_types]) + issues_list = [_() for _ in issues_types if not issue_class or _.__name__ in issue_class] + logger.debug("List of issues: %r", issues_list) logger.debug("Repository: %r", repo) # Configure Table table = Table(title=f"Check Issue Report of Repo [bold]{repository}[/bold]", @@ -182,6 +200,13 @@ def test(config, issue_file, issue_class, write, repository, output_path=None): ", ".join(issue.labels)) proposed_files.extend(issue_files) console.print(table) + + # show optional messages + console.print("\n\n") + for issue in issues: + for message in issue._messages: + console.print(f"[{message.type.value}]{message.type.name}:[/{message.type.value}] {message.text}") + console.print("\n\n") except Exception as e: logger.exception(e) error_console.print(str(e)) diff --git a/cli/client/utils.py b/cli/client/utils.py index 0ef2a95f0..affd0152e 100644 --- a/cli/client/utils.py +++ b/cli/client/utils.py @@ -40,15 +40,17 @@ def is_url(value): return False -def get_repository(repository: str, local_path: str = None): +def get_repository(repository: str, local_path: str): assert repository, repository if is_url(repository): remote_repo_url = repository if remote_repo_url.endswith('.git'): - return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=True, local_path=local_path) + return GithubWorkflowRepository.from_url(remote_repo_url, auto_cleanup=False, local_path=local_path) else: - return LocalWorkflowRepository(repository) - return ValueError("Repository type not supported") + local_copy_path = os.path.join(local_path, os.path.basename(repository)) + shutil.copytree(repository, local_copy_path) + return LocalWorkflowRepository(local_copy_path) + raise ValueError("Repository type not supported") def init_output_path(output_path): diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 96e8bc931..25a6fb2f8 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -31,11 +31,14 @@ services: depends_on: - "db" - "init" + - "redis" + user: "${USER_UID}:${USER_GID}" env_file: *env_file environment: - "FLASK_ENV=production" - "POSTGRESQL_HOST=db" - "POSTGRESQL_PORT=5432" + - "WEBSOCKET_SERVER=false" volumes: - "./certs:/certs:ro" - "./instance:/lm/instance:ro" @@ -82,7 +85,13 @@ services: - "db" - "init" - "redis" + user: "${USER_UID}:${USER_GID}" env_file: *env_file + environment: + - "FLASK_ENV=production" + - "POSTGRESQL_HOST=db" + - "POSTGRESQL_PORT=5432" + - "WEBSOCKET_SERVER=false" volumes: - "./certs:/certs:ro" - "./instance:/lm/instance:ro" @@ -91,6 +100,32 @@ services: networks: - life_monitor + ws_server: + image: crs4/lifemonitor + entrypoint: /usr/local/bin/wss-entrypoint.sh + restart: "always" + depends_on: + - "db" + - "init" + - "redis" + user: "${USER_UID}:${USER_GID}" + env_file: *env_file + environment: + - "FLASK_ENV=production" + - "WEBSOCKET_SERVER=true" + - "POSTGRESQL_HOST=db" + - "POSTGRESQL_PORT=5432" + - "LIFEMONITOR_TLS_KEY=/certs/lm.key" + - "LIFEMONITOR_TLS_CERT=/certs/lm.crt" + volumes: + - "./certs:/certs:ro" + - "./instance:/lm/instance:ro" + - "./settings.conf:/lm/settings.conf:ro" # default settings + - "data_workflows:/var/data/lm" + ports: + - "8001:8000" + networks: + - life_monitor redis: image: bitnami/redis:6.2 @@ -111,6 +146,7 @@ volumes: data_redis: data_workflows: + networks: life_monitor: # You can easily connect this docker-compose with a diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3f5e1ff9d..c822af048 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -35,6 +35,18 @@ services: worker: user: "${USER_UID}:${USER_GID}" + environment: + - "FLASK_ENV=development" + - "WORKER_PROCESSES=1" + - "WORKER_THREADS=1" + volumes: + - "./:/lm" + - "/lm/lifemonitor/static/dist" + + ws_server: + user: "${USER_UID}:${USER_GID}" + environment: + - "FLASK_ENV=development" volumes: - "./:/lm" - "/lm/lifemonitor/static/dist" @@ -42,6 +54,8 @@ services: dev_proxy: image: bitnami/nginx:1.19-debian-10 depends_on: + - "db" + - "redis" - "lm" ports: - "8000:8443" @@ -77,3 +91,7 @@ services: - "./certs:/certs:ro" networks: - life_monitor + + prometheus: + volumes: + - "./prometheus.dev.yml:/etc/prometheus/prometheus.yml" diff --git a/docker-compose.prom.yml b/docker-compose.monitoring.yml similarity index 100% rename from docker-compose.prom.yml rename to docker-compose.monitoring.yml diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index af98fd45b..fb1ee7a9f 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -37,16 +37,20 @@ WORKDIR /lm COPY \ docker/wait-for-postgres.sh \ docker/wait-for-redis.sh \ + docker/wait-for-file.sh \ docker/lm_entrypoint.sh \ docker/worker_entrypoint.sh \ + docker/wss-entrypoint.sh \ /usr/local/bin/ # Update permissions and install optional certificates RUN chmod 755 \ /usr/local/bin/wait-for-postgres.sh \ /usr/local/bin/wait-for-redis.sh \ + /usr/local/bin/wait-for-file.sh \ /usr/local/bin/lm_entrypoint.sh \ /usr/local/bin/worker_entrypoint.sh \ + /usr/local/bin/wss-entrypoint.sh \ /nextflow \ && certs=$(ls *.crt 2> /dev/null) \ && mv *.crt /usr/local/share/ca-certificates/ \ @@ -71,7 +75,7 @@ RUN git config --global user.name "LifeMonitor[bot]" \ && git config --global user.email "noreply@lifemonitor.eu" # Copy lifemonitor app -COPY --chown=lm:lm app.py lm-admin lm gunicorn.conf.py /lm/ +COPY --chown=lm:lm app.py ws.py lm-metrics-server lm-admin lm gunicorn.conf.py /lm/ COPY --chown=lm:lm specs /lm/specs COPY --chown=lm:lm lifemonitor /lm/lifemonitor COPY --chown=lm:lm migrations /lm/migrations diff --git a/docker/lm_entrypoint.sh b/docker/lm_entrypoint.sh index bac67640b..a2c7bce23 100644 --- a/docker/lm_entrypoint.sh +++ b/docker/lm_entrypoint.sh @@ -9,14 +9,21 @@ export KEY=${LIFEMONITOR_TLS_KEY:-/certs/lm.key} export CERT=${LIFEMONITOR_TLS_CERT:-/certs/lm.crt} export GUNICORN_CONF="${GUNICORN_CONF:-/lm/gunicorn.conf.py}" -printf "Waiting for postgresql...\n" >&2 +# wait for services wait-for-postgres.sh -printf "DB is ready. Starting application\n" >&2 +wait-for-redis.sh + if [[ "${FLASK_ENV}" == "development" || "${FLASK_ENV}" == "testingSupport" ]]; then printf "Staring app in DEV mode (Flask built-in web server with auto reloading)" python "${HOME}/app.py" else - export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d /tmp/lifemonitor_prometheus_multiproc_dir.XXXXXXXX) + PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR:-} + if [[ -z ${PROMETHEUS_MULTIPROC_DIR} ]]; then + metrics_base_path="/tmp/lifemonitor/metrics" + mkdir -p ${metrics_base_path} + export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d ${metrics_base_path}/backend.XXXXXXXX) + fi + export GUNICORN_SERVER="true" gunicorn --workers "${GUNICORN_WORKERS}" \ --threads "${GUNICORN_THREADS}" \ --config "${GUNICORN_CONF}" \ diff --git a/docker/nginx.dev.conf b/docker/nginx.dev.conf index f3fce13e2..3519c1f7f 100644 --- a/docker/nginx.dev.conf +++ b/docker/nginx.dev.conf @@ -5,6 +5,12 @@ upstream lm_app { server lm:8000 fail_timeout=0; } +# define connection_upgrade +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 8443 ssl default_server; client_max_body_size 4G; @@ -19,6 +25,18 @@ server { # force HTTP traffic to HTTPS error_page 497 https://$host:8443$request_uri; + location /socket.io/ { + proxy_pass https://lm_app; + proxy_http_version 1.1; + proxy_pass_header Server; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host lm:8000; + } + # set proxy location location / { # resolver 127.0.0.11 ipv6=off valid=30s; diff --git a/docker/wait-for-file.sh b/docker/wait-for-file.sh new file mode 100755 index 000000000..4bca71d41 --- /dev/null +++ b/docker/wait-for-file.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -o nounset +set -o errexit + +function debug_log() { + if [[ -n "${DEBUG:-}" ]]; then + log "${@}" + fi +} + +function log() { + printf "%s [wait-for-file] %s\n" "$(date +"%F %T")" "${*}" >&2 +} + +file="${1:-}" +timeout=${2:-0} + +if [[ -z ${file} ]]; then + log "You need to provide a filename" + exit 1 +fi + +log "Waiting for file ${file} (timeout: ${timeout})..." + +if [[ -e "$file" ]]; then + log "File ${file} found!" + exit 0 +fi + +if ((timeout > 0)); then + end_time=$((SECONDS + timeout)) + while [[ ! -e "$file" && $SECONDS -lt $end_time ]]; do + debug_log "File not found ${file} found... retry within 1 sec" + sleep 1 + done +else + while [[ ! -e "$file" ]]; do + debug_log "File not found ${file} found... retry within 1 sec" + sleep 1 + done +fi + +if [[ -e "$file" ]]; then + log "File ${file} found!" + exit 0 +else + log "File ${file} not found!" + exit 1 +fi + diff --git a/docker/wait-for-postgres.sh b/docker/wait-for-postgres.sh old mode 100644 new mode 100755 diff --git a/docker/worker_entrypoint.sh b/docker/worker_entrypoint.sh index d60de1564..cdc5544df 100755 --- a/docker/worker_entrypoint.sh +++ b/docker/worker_entrypoint.sh @@ -13,18 +13,28 @@ function log() { printf "%s [worker_entrypoint] %s\n" "$(date +"%F %T")" "${*}" >&2 } -# DEBUG="${DEBUG:-0}" +# wait for services +wait-for-postgres.sh +wait-for-redis.sh + +# set DEBUG flag +DEBUG="${DEBUG:-}" FLASK_ENV="${FLASK_ENV:-production}" -if [[ "${FLASK_ENV}" == "development" ]]; then +if [[ -z "${DEBUG}" && "${FLASK_ENV}" == "development" ]]; then DEBUG="${DEBUG:-1}" fi -# Create a directory for the worker's prometheus client. -# We follow the instructions in the dramatiq documentation -# https://dramatiq.io/advanced.html#gotchas-with-prometheus -export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d /tmp/lm_dramatiq_prometheus_multiproc_dir.XXXXXXXX) -rm -rf "${PROMETHEUS_MULTIPROC_DIR}/*" +# Create a directory for the worker's prometheus client if it doesn't exist yet +PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR:-} +if [[ -z ${PROMETHEUS_MULTIPROC_DIR} ]]; then + metrics_base_path="/tmp/lifemonitor/metrics" + mkdir -p ${metrics_base_path} + export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d ${metrics_base_path}/worker.XXXXXXXX) +fi + # dramatiq looks at the following two env variables +# ( instructions in the dramatiq documentation +# https://dramatiq.io/advanced.html#gotchas-with-prometheus ) export prometheus_multiproc_dir="${PROMETHEUS_MULTIPROC_DIR}" export dramatiq_prom_db="${PROMETHEUS_MULTIPROC_DIR}" @@ -32,8 +42,12 @@ log "Starting task queue worker container" debug_log "PROMETHEUS_MULTIPROC_DIR = ${PROMETHEUS_MULTIPROC_DIR}" if [[ -n "${DEBUG:-}" ]]; then - watch='--watch .' verbose='--verbose' + log "Debug Mode Enabled" +fi + +if [[ ${FLASK_ENV} == "development" ]]; then + watch='--watch .' log "Worker watching source code directory" fi @@ -70,6 +84,7 @@ while : ; do ${threads:-} \ lifemonitor.tasks.worker:broker lifemonitor.tasks ${queues} exit_code=$? + exit_code=$? if [[ $exit_code == 3 ]]; then log "dramatiq worker could not connect to message broker (exit code ${exit_code})" log "Restarting..." @@ -80,4 +95,4 @@ done log "Worker exiting with exit code ${exit_code}" -exit ${exit_code} +exit ${exit_code} \ No newline at end of file diff --git a/docker/wss-entrypoint.sh b/docker/wss-entrypoint.sh new file mode 100755 index 000000000..278e3125c --- /dev/null +++ b/docker/wss-entrypoint.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -o nounset +set -o errexit + +export POSTGRESQL_USERNAME="${POSTGRESQL_USERNAME:-lm}" +export POSTGRESQL_DATABASE="${POSTGRESQL_DATABASE:-lm}" +export KEY=${LIFEMONITOR_TLS_KEY:-/certs/lm.key} +export CERT=${LIFEMONITOR_TLS_CERT:-/certs/lm.crt} + +# set websocket server port +export WEBSOCKET_SERVER_PORT=${WEBSOCKET_SERVER_PORT:-8001} + +# TODO: check if this flag is needed +export WEBSOCKET_SERVER="false" + +# set switch to enable autoreload feature of gunicorn +export WEBSOCKET_SERVER_ENV=${WEBSOCKET_SERVER_ENV:-${FLASK_ENV:-production}} + +# wait for services +wait-for-postgres.sh +wait-for-redis.sh + +# Create a directory for the worker's prometheus client if it doesn't exist yet +PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR:-} +if [[ -z ${PROMETHEUS_MULTIPROC_DIR} ]]; then + metrics_base_path="/tmp/lifemonitor/metrics" + mkdir -p ${metrics_base_path} + export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d ${metrics_base_path}/websocket-server.XXXXXXXX) +fi + +# start gunicorn server +export GUNICORN_SERVER="true" +reload_opt="" +if [[ "${WEBSOCKET_SERVER_ENV}" == "development" ]]; then reload_opt="--reload"; fi +gunicorn -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 \ + ${reload_opt} \ + --certfile="${CERT}" --keyfile="${KEY}" \ + -b 0.0.0.0:${WEBSOCKET_SERVER_PORT} ws + + diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index d394ef051..053a2d701 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -19,8 +19,14 @@ spec: - name: lifemonitor-init image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} - command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin db init && ./lm-admin task-queue reset"] + command: + - /bin/sh + - -c + - | + wait-for-redis.sh \ + && wait-for-postgres.sh \ + && ./lm-admin db init \ + && ./lm-admin task-queue reset env: {{- include "lifemonitor.common-env" . | nindent 10 }} volumeMounts: diff --git a/k8s/templates/secret.yaml b/k8s/templates/secret.yaml index 20fb70747..8e372f330 100644 --- a/k8s/templates/secret.yaml +++ b/k8s/templates/secret.yaml @@ -106,6 +106,14 @@ stringData: {{- end }} {{- end }} + {{- if .Values.identity_providers.lsaai }} + {{- if and .Values.identity_providers.lsaai.client_id .Values.identity_providers.lsaai.client_secret }} + # LifeScience OAuth2 settings + LSAAI_CLIENT_ID="{{ .Values.identity_providers.lsaai.client_id }}" + LSAAI_CLIENT_SECRET="{{ .Values.identity_providers.lsaai.client_secret }}" + {{- end }} + {{- end }} + # Set tokens for testingService {{- if .Values.testing_services -}} {{- range $k, $v := .Values.testing_services }} diff --git a/k8s/templates/service.yaml b/k8s/templates/service.yaml index ac8c8ccf6..0aec5968c 100644 --- a/k8s/templates/service.yaml +++ b/k8s/templates/service.yaml @@ -14,3 +14,25 @@ spec: selector: {{- include "chart.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: backend + +--- + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "chart.fullname" . }}-wss + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + type: {{ .Values.lifemonitor.service.type }} + ports: + - port: 8001 + targetPort: 8001 + protocol: TCP + name: wss + selector: + {{- include "chart.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: wss + +--- + diff --git a/k8s/templates/wss-deployment.yaml b/k8s/templates/wss-deployment.yaml new file mode 100644 index 000000000..82bb88699 --- /dev/null +++ b/k8s/templates/wss-deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "chart.fullname" . }}-wss + labels: + {{- include "chart.labels" . | nindent 4 }} + app.kubernetes.io/component: wss +spec: + {{- if not .Values.lifemonitor.autoscaling.enabled }} + replicas: {{ .Values.lifemonitor.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "chart.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: wss + template: + metadata: + annotations: + checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.lifemonitor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "chart.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: wss + # prometheus.io/scrape: 'true' + # prometheus.io/path: 'metrics' + # prometheus.io/port: '9090' + # prometheus.io/scheme: 'http' + spec: + {{- with .Values.lifemonitor.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "chart.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }} + initContainers: + - name: init + securityContext: + {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + command: ["/bin/sh","-c"] + # args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin db wait-for-db && wait-for-file.sh ${PROMETHEUS_MULTIPROC_DIR}"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin db wait-for-db"] + env: + {{- include "lifemonitor.common-env" . | nindent 12 }} + volumeMounts: + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} + containers: + - name: wss + command: ["/bin/sh","-c"] + args: ["wss-entrypoint.sh"] + securityContext: + {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + env: + {{- include "lifemonitor.common-env" . | nindent 12 }} + volumeMounts: + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} + ports: + - name: wss + containerPort: 8001 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + # livenessProbe: + # httpGet: + # scheme: HTTPS + # path: /health + # port: 8001 + # readinessProbe: + # httpGet: + # scheme: HTTPS + # path: /health + # port: 8001 + # initialDelaySeconds: 5 + # periodSeconds: 3 + resources: + {{- toYaml .Values.lifemonitor.resources | nindent 12 }} + volumes: + {{- include "lifemonitor.common-volume" . | nindent 8 }} + {{- with .Values.lifemonitor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/k8s/values.yaml b/k8s/values.yaml index 5c8ba391b..cf2fd19fc 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -209,12 +209,18 @@ worker: queues: - name: github # image: *lifemonitorImage - - name: heartbeat + # - name: heartbeat # image: *lifemonitorImage - name: notifications # image: *lifemonitorImage - name: builds # image: *lifemonitorImage + - name: workflows + # image: *lifemonitorImage + # - name: metrics + # image: *lifemonitorImage + - name: ws + # image: *lifemonitorImage podAnnotations: {} diff --git a/lifemonitor/api/controllers.py b/lifemonitor/api/controllers.py index 52f944711..5cdb8887f 100644 --- a/lifemonitor/api/controllers.py +++ b/lifemonitor/api/controllers.py @@ -20,11 +20,13 @@ import logging import tempfile +from typing import Optional import connexion -import lifemonitor.exceptions as lm_exceptions import werkzeug -from flask import Response, request, render_template +from flask import Response, current_app, redirect, render_template, request + +import lifemonitor.exceptions as lm_exceptions from lifemonitor.api import models, serializers from lifemonitor.api.services import LifeMonitor from lifemonitor.auth import (EventType, authorized, current_registry, @@ -35,6 +37,8 @@ OAuthIdentityNotFoundException from lifemonitor.cache import Timeout, cached, clear_cache from lifemonitor.lang import messages +from lifemonitor.tasks.models import Job +from lifemonitor.utils import notify_updates, notify_workflow_version_updates # Initialize a reference to the LifeMonitor instance lm = LifeMonitor.get_instance() @@ -104,14 +108,17 @@ def workflow_registries_get_current(): @cached(timeout=Timeout.REQUEST) -def workflows_get(status=False, versions=False): +def workflows_get(status=False, versions=False, subscriptions=False, only_subscriptions=False): workflows = lm.get_public_workflows() if current_user and not current_user.is_anonymous: - workflows.extend(lm.get_user_workflows(current_user)) + workflows.extend(lm.get_user_workflows(current_user, + include_subscriptions=subscriptions, + only_subscriptions=only_subscriptions)) elif current_registry: workflows.extend(lm.get_registry_workflows(current_registry)) logger.debug("workflows_get. Got %s workflows (user: %s)", len(workflows), current_user) - return serializers.ListOfWorkflows(workflow_status=status, workflow_versions=versions).dump( + return serializers.ListOfWorkflows(workflow_status=status, workflow_versions=versions, + subscriptionsOf=[current_user] if subscriptions else []).dump( list(dict.fromkeys(workflows)) ) @@ -150,7 +157,7 @@ def workflows_get_by_id(wf_uuid, wf_version): return response if isinstance(response, Response) \ else serializers.WorkflowVersionSchema(subscriptionsOf=[current_user] if not current_user.is_anonymous - else None).dump(response) + else None, rocrate_metadata=True).dump(response) @cached(timeout=Timeout.REQUEST) @@ -183,9 +190,8 @@ def workflows_get_versions_by_id(wf_uuid): @cached(timeout=Timeout.REQUEST) -def workflows_get_status(wf_uuid): - wf_version = request.args.get('version', 'latest').lower() - response = __get_workflow_version__(wf_uuid, wf_version) +def workflows_get_status(wf_uuid, version): + response = __get_workflow_version__(wf_uuid, version) return response if isinstance(response, Response) \ else serializers.WorkflowStatusSchema().dump(response) @@ -259,15 +265,16 @@ def registry_user_workflows_post(user_id, body): @authorized @cached(timeout=Timeout.REQUEST) -def user_workflows_get(status=False, subscriptions=False, versions=False): +def user_workflows_get(status=False, versions=False, subscriptions=False, only_subscriptions: bool = False): if not current_user or current_user.is_anonymous: return lm_exceptions.report_problem(401, "Unauthorized", detail=messages.no_user_in_session) - workflows = lm.get_user_workflows(current_user, include_subscriptions=subscriptions) + workflows = lm.get_user_workflows(current_user, + include_subscriptions=subscriptions, + only_subscriptions=only_subscriptions) logger.debug("user_workflows_get. Got %s workflows (user: %s)", len(workflows), current_user) return serializers.ListOfWorkflows(workflow_status=status, workflow_versions=versions, - subscriptionsOf=[current_user] - if subscriptions else None).dump(workflows) + subscriptionsOf=[current_user] if subscriptions else None).dump(workflows) @authorized @@ -388,13 +395,17 @@ def __check_submitter_and_registry__(body, _registry=None, _submitter_id=None, _ submitter = current_user if current_user and not current_user.is_anonymous else None if not submitter: try: - submitter_id = body.get('submitter_id', _submitter_id) - if submitter_id: - # Try to find the identity of the submitter - identity = lm.find_registry_user_identity(registry, - internal_id=current_user.id, - external_id=submitter_id) - submitter = identity.user + user_id = body.get('user_id', current_user.id if current_user else None) + if user_id: + submitter = lm.get_user_by_id(user_id) + if not submitter: + submitter_id = body.get('submitter_id', _submitter_id) + if submitter_id: + # Try to find the identity of the submitter + identity = lm.find_registry_user_identity(registry, + internal_id=user_id, + external_id=submitter_id) + submitter = identity.user except KeyError: # return lm_exceptions.report_problem(400, "Bad request", # detail=messages.no_submitter_id_provided) @@ -411,7 +422,13 @@ def __check_submitter_and_registry__(body, _registry=None, _submitter_id=None, _ @authorized -def workflows_post(body, _registry=None, _submitter_id=None): +def workflows_post(*args, **kwargs): + return process_workflows_post(*args, **kwargs) + + +def process_workflows_post(body, _registry=None, _submitter_id=None, + async_processing: Optional[bool] = None, job: Job = None): + logger.warning("The current body: %r", body) # check if there exists a submitter and/or a registry in the current request registry, submitter = __check_submitter_and_registry__(body, _registry, _submitter_id) # extract roc_link or rocrate from the request @@ -420,6 +437,38 @@ def workflows_post(body, _registry=None, _submitter_id=None): if not registry and not roc_link and not encoded_rocrate: return lm_exceptions.report_problem(400, "Bad Request", extra_info={"missing input": "roc_link OR rocrate"}, detail=messages.input_data_missing) + + # check whether to handle the registration asynchronously + async_processing = body.get('async', False) if async_processing is None else async_processing + if async_processing: + # collect registration data + registration_data = { + "submitter_id": submitter.id, + "user_id": submitter.id, + "version": body['version'], + "uuid": body.get('uuid', None), + "identifier": body.get('identifier', None), + "name": body.get('name', None), + "authorization": body.get('authorization', None), + "public": body.get('public', False) + } + if roc_link: + registration_data['roc_link'] = roc_link + if encoded_rocrate: + registration_data['rocrate'] = encoded_rocrate + if registry: + registration_data["registry"] = registry.server_credentials.client_name + # create async Job + job = Job(job_type='workflow_registration', # job_name='register_workflow', + status='waiting', + listening_rooms=[str(registration_data['submitter_id'])]) + job.update_data({ + 'data': registration_data + }) + job.save() + job.submit(current_app, as_job_name='register_workflow') + return redirect(f'/jobs/status/{job.id}', code=302) + # register workflow through the 'lm' service try: w = lm.register_workflow( @@ -431,11 +480,17 @@ def workflows_post(body, _registry=None, _submitter_id=None): workflow_registry=registry, name=body.get('name', None), authorization=body.get('authorization', None), - public=body.get('public', False) + public=body.get('public', False), + job=job ) logger.debug("workflows_post. Created workflow '%s' (ver.%s)", w.uuid, w.version) clear_cache() - return {'uuid': str(w.workflow.uuid), 'wf_version': w.version, 'name': w.name}, 201 + notify_workflow_version_updates([w], type='sync', delay=2) + return {'uuid': str(w.workflow.uuid), 'wf_version': w.version, 'name': w.name, + 'meta': { + 'created': w.workflow.created.timestamp(), + 'modified': w.workflow.modified.timestamp()} + }, 201 except KeyError as e: return lm_exceptions.report_problem(400, "Bad Request", extra_info={"exception": str(e)}, detail=messages.input_data_missing) @@ -470,8 +525,17 @@ def workflows_put(wf_uuid, body): # update basic information aboud the specified workflow workflow_version.workflow.name = body.get('name', workflow_version.workflow.name) workflow_version.workflow.public = body.get('public', workflow_version.workflow.public) - workflow_version.workflow.save() - return connexion.NoContent, 204 + # workflow_version.workflow.save() + workflow_version.save() + clear_cache() + notify_workflow_version_updates([workflow_version], type='sync', delay=2) + return { + 'meta': + { + 'created': workflow_version.created.timestamp(), + 'modified': workflow_version.modified.timestamp() + } + }, 201 @authorized @@ -499,9 +563,15 @@ def workflows_version_put(wf_uuid, wf_version, body): ) clear_cache() if updated_workflow_version.uuid != workflow_version.uuid: - return {'uuid': str(updated_workflow_version.workflow.uuid), - 'wf_version': updated_workflow_version.version, - 'name': updated_workflow_version.name}, 201 + return { + 'uuid': str(updated_workflow_version.workflow.uuid), + 'wf_version': updated_workflow_version.version, + 'name': updated_workflow_version.name, + 'meta': { + 'created': updated_workflow_version.workflow.created.timestamp(), + 'modified': updated_workflow_version.workflow.modified.timestamp() + } + }, 201 return connexion.NoContent, 204 except KeyError as e: @@ -528,6 +598,8 @@ def workflows_version_put(wf_uuid, wf_version, body): @authorized def workflows_delete_version(wf_uuid, wf_version): try: + # get a reference to the workflow version to be updated + workflow_version = __get_workflow_version__(wf_uuid, wf_version=wf_version) if current_user and not current_user.is_anonymous: lm.deregister_user_workflow_version(wf_uuid, wf_version, current_user) elif current_registry: @@ -536,6 +608,7 @@ def workflows_delete_version(wf_uuid, wf_version): return lm_exceptions.report_problem(403, "Forbidden", detail=messages.no_user_in_session) clear_cache() + notify_workflow_version_updates([workflow_version], type='delete') return connexion.NoContent, 204 except OAuthIdentityNotFoundException as e: return lm_exceptions.report_problem(401, "Unauthorized", extra_info={"exception": str(e)}) @@ -551,6 +624,8 @@ def workflows_delete_version(wf_uuid, wf_version): @authorized def workflows_delete(wf_uuid): try: + w = lm.get_workflow(wf_uuid) + versions = [{'uuid': wf_uuid, 'version': w.version} for w in w.versions.values()] if current_user and not current_user.is_anonymous: lm.deregister_user_workflow(wf_uuid, current_user) elif current_registry: @@ -559,12 +634,13 @@ def workflows_delete(wf_uuid): return lm_exceptions.report_problem(403, "Forbidden", detail=messages.no_user_in_session) clear_cache() + notify_updates(versions, type='delete') return connexion.NoContent, 204 except OAuthIdentityNotFoundException as e: return lm_exceptions.report_problem(401, "Unauthorized", extra_info={"exception": str(e)}) except lm_exceptions.EntityNotFoundException as e: return lm_exceptions.report_problem(404, "Not Found", extra_info={"exception": str(e.detail)}, - detail=messages.workflow_version_not_found.format(wf_uuid)) + detail=messages.workflow_not_found.format(wf_uuid)) except lm_exceptions.NotAuthorizedException as e: return lm_exceptions.report_problem(403, "Forbidden", extra_info={"exception": str(e)}) except Exception as e: @@ -589,9 +665,15 @@ def workflows_get_issue_types_as_html(back=None): def workflows_get_suites(wf_uuid, version='latest', status: bool = False, latest_builds: bool = False): workflow_version = __get_workflow_version__(wf_uuid, version) logger.debug("GET suites of workflow version: %r", workflow_version) - return serializers.ListOfSuites( - status=status, latest_builds=latest_builds - ).dump(workflow_version.test_suites) + try: + return serializers.ListOfSuites( + status=status, latest_builds=latest_builds + ).dump(workflow_version.test_suites) + finally: + if latest_builds: + for s in workflow_version.test_suites: + for i in s.test_instances: + i.save() def _get_suite_or_problem(suite_uuid): @@ -624,16 +706,48 @@ def _get_suite_or_problem(suite_uuid): @cached(timeout=Timeout.REQUEST) def suites_get_by_uuid(suite_uuid, status: bool = False, latest_builds: bool = False): - response = _get_suite_or_problem(suite_uuid) - return response if isinstance(response, Response) \ - else serializers.SuiteSchema(status=status, latest_builds=latest_builds).dump(response) + suite = _get_suite_or_problem(suite_uuid) + try: + return suite if isinstance(suite, Response) \ + else serializers.SuiteSchema(status=status, latest_builds=latest_builds).dump(suite) + except Exception as e: + return __handle_testing_service_error__(e, instance_uuid=suite_uuid, + message=messages.suite_status_unavailable.format(suite_uuid) + if isinstance(e, lm_exceptions.TestingServiceException) else str(e)) + finally: + if suite and latest_builds: + for i in suite.test_instances: + i.save() @cached(timeout=Timeout.REQUEST) def suites_get_status(suite_uuid): - response = _get_suite_or_problem(suite_uuid) - return response if isinstance(response, Response) \ - else serializers.SuiteStatusSchema().dump(response) + suite = _get_suite_or_problem(suite_uuid) + try: + return suite if isinstance(suite, Response) \ + else serializers.SuiteStatusSchema().dump(suite) + except Exception as e: + return __handle_testing_service_error__(e, instance_uuid=suite_uuid, + message=messages.suite_status_unavailable.format(suite_uuid)) + finally: + if suite: + for i in suite.test_instances: + i.save() + + +def __handle_testing_service_error__(e: Exception, instance_uuid: str = None, message: str = None): + extra_info = {} + if isinstance(e, lm_exceptions.TestingServiceException): + extra_info['testing_service_error'] = { + "title": e.title, + "status": e.status, + "message": e.detail, + } + else: + extra_info["exception"] = str(e) + return lm_exceptions.report_problem(500, "Unavailable builds", instance=instance_uuid, + detail=message, + extra_info=extra_info) @cached(timeout=Timeout.REQUEST) @@ -791,6 +905,26 @@ def instances_delete_by_id(instance_uuid): @cached(timeout=Timeout.REQUEST) def instances_get_builds(instance_uuid, limit): + # response = _get_instances_or_problem(instance_uuid) + # logger.debug("Number of builds to load: %r", limit) + # try: + # builds = response.get_test_builds(limit=limit) + # response.save() + # return response if isinstance(response, Response) \ + # else serializers.ListOfTestBuildsSchema().dump(builds) + # except Exception as e: + # extra_info = {} + # if isinstance(e, lm_exceptions.TestingServiceException): + # extra_info['testing_service_error'] = { + # "title": e.title, + # "status": e.status, + # "message": e.detail, + # } + # else: + # extra_info["exception"] = str(e) + # return lm_exceptions.report_problem(500, "Unavailable builds", instance=instance_uuid, + # detail=messages.instance_builds_unavailable.format(instance_uuid), + # extra_info=extra_info) response = _get_instances_or_problem(instance_uuid) logger.info("Number of builds to load: %r", limit) return response if isinstance(response, Response) \ diff --git a/lifemonitor/api/models/issues/__init__.py b/lifemonitor/api/models/issues/__init__.py index 4d4f9399e..3a99277fb 100644 --- a/lifemonitor/api/models/issues/__init__.py +++ b/lifemonitor/api/models/issues/__init__.py @@ -24,17 +24,46 @@ import inspect import logging import os +from enum import Enum from hashlib import sha1 from importlib import import_module from pathlib import Path from typing import List, Optional, Type import networkx as nx + from lifemonitor.api.models import repositories # set module level logger logger = logging.getLogger(__name__) +ROOT_ISSUE = 'r' + + +class IssueMessage: + + class TYPE(Enum): + INFO = "info" + WARNING = "warning" + ERROR = "error" + + def __init__(self, type: TYPE, text: str) -> None: + self._type = type + self._text = text + + def __eq__(self, other): + if isinstance(other, IssueMessage): + return self.type == other.type and self.text == other.text + return False + + @property + def type(self) -> TYPE: + return self._type + + @property + def text(self) -> str: + return self._text + class WorkflowRepositoryIssue(): @@ -47,6 +76,7 @@ class WorkflowRepositoryIssue(): def __init__(self): self._changes = [] + self._messages: List[IssueMessage] = [] @property def id(self) -> str: @@ -86,6 +116,18 @@ def get_changes(self, repo: repositories.WorkflowRepository) -> List[repositorie def has_changes(self) -> bool: return bool(self._changes) and len(self._changes) > 0 + def add_message(self, message: IssueMessage): + self._messages.append(message) + + def remove_message(self, message: IssueMessage): + self._messages.remove(message) + + def get_messages(self) -> List[IssueMessage]: + return self._messages + + def has_messages(self) -> bool: + return len(self._messages) > 0 + @classmethod def get_identifier(cls) -> str: return cls.to_string(cls) @@ -134,8 +176,9 @@ def generate_template(class_name: str, name: str, description: str = "", depends def load_issue(issue_file) -> List[Type[WorkflowRepositoryIssue]]: issues = {} - base_module = '{}'.format(os.path.join(os.path.dirname(issue_file)).replace('/', '.')) + base_module = '{}'.format(os.path.join(os.path.dirname(issue_file)).replace('./', '').replace('/', '.')) m = '{}.{}'.format(base_module, os.path.basename(issue_file)[:-3]) + logger.debug("BaseModule: %r -- Module: %r" % (base_module, m)) mod = import_module(m) for _, obj in inspect.getmembers(mod): if inspect.isclass(obj) \ @@ -146,9 +189,8 @@ def load_issue(issue_file) -> List[Type[WorkflowRepositoryIssue]]: return issues.values() -def find_issue_types(path: Optional[str] = None) -> List[Type[WorkflowRepositoryIssue]]: +def get_issue_graph(path: Optional[str] = None) -> nx.DiGraph: errors = [] - issues = {} g = nx.DiGraph() base_path = Path(path) if path else Path(__file__).parent @@ -168,13 +210,12 @@ def find_issue_types(path: Optional[str] = None) -> List[Type[WorkflowRepository and inspect.getmodule(obj) == mod \ and obj != WorkflowRepositoryIssue \ and issubclass(obj, WorkflowRepositoryIssue): - issues[obj.__name__] = obj dependencies = getattr(obj, 'depends_on', None) if not dependencies or len(dependencies) == 0: - g.add_edge('r', obj.__name__) + g.add_edge(ROOT_ISSUE, obj) else: for dep in dependencies: - g.add_edge(dep.__name__, obj.__name__) + g.add_edge(dep, obj) except ModuleNotFoundError as e: logger.exception(e) logger.error("ModuleNotFoundError: Unable to load module %s", m) @@ -183,10 +224,11 @@ def find_issue_types(path: Optional[str] = None) -> List[Type[WorkflowRepository logger.error("** There were some errors loading application modules.**") if logger.isEnabledFor(logging.DEBUG): logger.error("** Unable to load issues from %s", ", ".join(errors)) - logger.debug("Issues: %r", [_.__name__ for _ in issues.values()]) - sorted_issues = [issues[_] for _ in nx.dfs_preorder_nodes(g, source='r') if _ != 'r'] - logger.debug("Sorted issues: %r", [_.__name__ for _ in sorted_issues]) - return sorted_issues + return g + + +def find_issue_types(path: Optional[str] = None) -> List[Type[WorkflowRepositoryIssue]]: + return [i for i in get_issue_graph(path).nodes if i != ROOT_ISSUE] __all__ = ["WorkflowRepositoryIssue"] diff --git a/lifemonitor/api/models/issues/general/lm.py b/lifemonitor/api/models/issues/general/lm.py index d59989ae4..0f389c252 100644 --- a/lifemonitor/api/models/issues/general/lm.py +++ b/lifemonitor/api/models/issues/general/lm.py @@ -22,8 +22,10 @@ import logging -from lifemonitor.api.models.issues import WorkflowRepositoryIssue +from lifemonitor.utils import get_validation_schema_url +from lifemonitor.api.models.issues import IssueMessage, WorkflowRepositoryIssue from lifemonitor.api.models.repositories import WorkflowRepository +from lifemonitor.schemas.validators import ValidationError, ValidationResult # set module level logger logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ class MissingLMConfigFile(WorkflowRepositoryIssue): name = "Missing LifeMonitor configuration file" description = "No lifemonitor.yaml configuration file found on this repository.
"\ "The lifemonitor.yaml should be placed on the root of this repository." - labels = ['config', 'enhancement'] + labels = ['lifemonitor'] def check(self, repo: WorkflowRepository) -> bool: if repo.config is None: @@ -41,3 +43,23 @@ def check(self, repo: WorkflowRepository) -> bool: self.add_change(config) return True return False + + +class InvalidConfigFile(WorkflowRepositoryIssue): + + name = "Invalid LifeMonitor configuration file" + description = "The LifeMonitor configuration file found on this repository "\ + f"is not valid according to the schema " \ + f"{get_validation_schema_url()}.
" + labels = ['lifemonitor'] + depends_on = [MissingLMConfigFile] + + def check(self, repo: WorkflowRepository) -> bool: + if repo.config is None: + return True + result: ValidationResult = repo.config.validate() + if not result.valid: + if isinstance(result, ValidationError): + self.add_message(IssueMessage(IssueMessage.TYPE.ERROR, result.error)) + return True + return False diff --git a/lifemonitor/api/models/issues/general/metadata.py b/lifemonitor/api/models/issues/general/metadata.py index cfdcb9542..05f2b3f75 100644 --- a/lifemonitor/api/models/issues/general/metadata.py +++ b/lifemonitor/api/models/issues/general/metadata.py @@ -32,13 +32,13 @@ class MissingWorkflowName(WorkflowRepositoryIssue): - name = "Missing property name for Workflow RO-Crate" + name = "Missing workflow name in metadata" description = "No name defined for this workflow.
You can set the workflow name on the `ro-crate-metadata.yaml` or `lifemonitor.yaml` file" - labels = ['invalid', 'bug'] + labels = ['metadata'] depends_on = [RepositoryNotInitialised] def check(self, repo: WorkflowRepository) -> bool: - if repo.config.workflow_name: + if repo.config and repo.config.workflow_name: return False if repo.metadata and repo.metadata.main_entity_name: return False diff --git a/lifemonitor/api/models/issues/general/repo_layout.py b/lifemonitor/api/models/issues/general/repo_layout.py index 927334e1a..d11f04560 100644 --- a/lifemonitor/api/models/issues/general/repo_layout.py +++ b/lifemonitor/api/models/issues/general/repo_layout.py @@ -24,7 +24,6 @@ from lifemonitor.api.models.issues import WorkflowRepositoryIssue from lifemonitor.api.models.repositories import WorkflowRepository -from .lm import MissingLMConfigFile # set module level logger logger = logging.getLogger(__name__) @@ -33,8 +32,7 @@ class RepositoryNotInitialised(WorkflowRepositoryIssue): name = "Repository not intialised" description = "No workflow and crate metadata found on this repository." - labels = ['invalid', 'enhancement', 'config'] - depends_on = [MissingLMConfigFile] + labels = ['best-practices'] def check(self, repo: WorkflowRepository) -> bool: return repo.find_workflow() is None and repo.metadata is None @@ -44,7 +42,7 @@ class MissingWorkflowFile(WorkflowRepositoryIssue): name = "Missing workflow file" description = "No workflow found on this repository.
"\ "You should place the workflow file (e.g., .ga file) according to the best practices ." - labels = ['invalid', 'bug'] + labels = ['best-practices'] depends_on = [RepositoryNotInitialised] def check(self, repo: WorkflowRepository) -> bool: @@ -55,7 +53,7 @@ class MissingROCrateFile(WorkflowRepositoryIssue): name = "Missing RO-Crate metadata" description = "No ro-crate-metadata.json found on this repository.
"\ "The ro-crate-metadata.json should be placed on the root of this repository." - labels = ['invalid', 'enhancement'] + labels = ['metadata'] depends_on = [MissingWorkflowFile] def check(self, repo: WorkflowRepository) -> bool: @@ -69,7 +67,7 @@ def check(self, repo: WorkflowRepository) -> bool: class MissingROCrateWorkflowFile(WorkflowRepositoryIssue): name = "Missing RO-Crate workflow file" description = "The workflow file declared on RO-Crate metadata is missing in this repository." - labels = ['invalid', 'bug'] + labels = ['metadata'] depends_on = [MissingROCrateFile] def check(self, repo: WorkflowRepository) -> bool: diff --git a/lifemonitor/api/models/repositories/base.py b/lifemonitor/api/models/repositories/base.py index ad53d03ec..d65e8ba97 100644 --- a/lifemonitor/api/models/repositories/base.py +++ b/lifemonitor/api/models/repositories/base.py @@ -27,7 +27,7 @@ import os from abc import abstractclassmethod from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Type import git import giturlparse @@ -153,28 +153,55 @@ def find_workflow(self) -> WorkflowFile: def contains(self, file: RepositoryFile) -> bool: return self.__contains__(self.files, file) + @staticmethod + def _issue_name_included(issue_name: str, + include_list: List[str] | None = None, + exclude_list: List[str] | None = None) -> bool: + if include_list and (issue_name in [to_camel_case(_) for _ in include_list]): + return True + + if exclude_list is None: + return True + + return issue_name not in [to_camel_case(_) for _ in exclude_list] + def check(self, fail_fast: bool = True, include=None, exclude=None) -> IssueCheckResult: found_issues = [] - checked = [] - for issue_type in issues.find_issue_types(): - if (not exclude or issue_type.__name__ not in [to_camel_case(_) for _ in exclude]) or \ - (not include or issue_type.__name__ in [to_camel_case(_) for _ in include]): + issue_graph = issues.get_issue_graph() + + checked = set() + visited = set() + queue = [i for i in issue_graph.neighbors(issues.ROOT_ISSUE) + if self._issue_name_included(i.__name__, include, exclude)] + while queue: + issue_type = queue.pop() + if issue_type not in visited: issue = issue_type() - to_be_solved = issue.check(self) - checked.append(issue) - if to_be_solved: + try: + failed = issue.check(self) + except Exception as e: + logger.error("Issue %s failed to run. It raised an exception: %s", + issue_type.__name__, e) + continue # skip this issue by not marking it as visited (otherwise it shows as "passed") + visited.add(issue_type) + checked.add(issue) + if not failed: + neighbors = [i for i in issue_graph.neighbors(issue_type) + if self._issue_name_included(i.__name__, include, exclude)] + queue.extend(neighbors) + else: found_issues.append(issue) if fail_fast: break - return IssueCheckResult(self, checked, found_issues) + return IssueCheckResult(self, list(checked), found_issues) @classmethod def __contains__(cls, files, file) -> bool: return cls.__find_file__(files, file) is not None @classmethod - def __find_file__(cls, files, file) -> RepositoryFile: + def __find_file__(cls, files, file) -> RepositoryFile | None: for f in files: if f == file or (f.name == file.name and f.dir == file.dir): return f @@ -186,8 +213,13 @@ def __file_pointer__(cls, f: RepositoryFile): if os.path.exists(f.path): fp = open(f.path, 'rb') else: - logger.debug("Reading file content: %r", f.get_content(binary_mode=False).encode()) - fp = io.BytesIO(f.get_content(binary_mode=False).encode()) + content = f.get_content(binary_mode=False) + if content: + content_str = content.encode() + else: + content_str = b"" + logger.debug("Reading file content: %r", content_str if content else b"None") + fp = io.BytesIO(content_str) return fp @classmethod @@ -310,6 +342,15 @@ def get_issue(self, issue_name: str) -> issues.WorkflowRepositoryIssue: def found_issues(self) -> bool: return len(self.issues) > 0 + def is_checked(self, issue: Type[issues.WorkflowRepositoryIssue] | issues.WorkflowRepositoryIssue) -> bool: + if issue and issue in self.checked: + return True + if isinstance(issue, issues.WorkflowRepositoryIssue): + for issue_type in self.checked: + if isinstance(issue, issue_type): + return True + return False + @property def solved(self) -> List[issues.WorkflowRepositoryIssue]: if not self._solved: diff --git a/lifemonitor/api/models/repositories/config.py b/lifemonitor/api/models/repositories/config.py index 1bc383fda..7ca4440bd 100644 --- a/lifemonitor/api/models/repositories/config.py +++ b/lifemonitor/api/models/repositories/config.py @@ -26,8 +26,11 @@ import os.path from typing import Dict, List, Optional -import lifemonitor.api.models as models import yaml + +import lifemonitor.api.models as models +from lifemonitor.schemas.validators import (ConfigFileValidator, + ValidationResult) from lifemonitor.utils import match_ref from .files import RepositoryFile, TemplateRepositoryFile @@ -72,6 +75,17 @@ def _raw_data(self) -> dict: self.__raw_data = {} return self.__raw_data + @property + def is_valid(self) -> bool: + try: + result: ValidationResult = ConfigFileValidator.validate(self.load()) + return result.valid + except Exception: + return False + + def validate(self) -> ValidationResult: + return ConfigFileValidator.validate(self.load()) + @property def workflow_name(self) -> str: return self._raw_data.get('name', None) @@ -175,7 +189,7 @@ def tags(self) -> List[str]: @classmethod def new(cls, repository_path: str, workflow_title: str = "Workflow RO-Crate", public: bool = False, main_branch: str = "main") -> WorkflowRepositoryConfig: tmpl = TemplateRepositoryFile(repository_path="lifemonitor/templates/repositories/base", name=cls.TEMPLATE_FILENAME) - registries = models.WorkflowRegistry.all() + registries = ["wfhub", "wfhubdev"] issue_types = models.WorkflowRepositoryIssue.all() os.makedirs(repository_path, exist_ok=True) tmpl.write(workflow_title=workflow_title, main_branch=main_branch, public=public, diff --git a/lifemonitor/api/models/repositories/files/templates.py b/lifemonitor/api/models/repositories/files/templates.py index cb7be45cd..cda3f029b 100644 --- a/lifemonitor/api/models/repositories/files/templates.py +++ b/lifemonitor/api/models/repositories/files/templates.py @@ -24,7 +24,8 @@ import os from typing import Dict, Optional -from flask import render_template_string +from jinja2 import Environment, FileSystemLoader, select_autoescape + from lifemonitor import utils from .base import RepositoryFile @@ -69,16 +70,17 @@ def get_content(self, binary_mode: bool = False, refresh: bool = False, **kwargs data.update(kwargs) if (not self._content or refresh) and self.dir: is_template = self.template_file_path.endswith('.j2') - with open(self.template_file_path, 'rb' if binary_mode and not is_template else 'r') as f: - content = f.read() - if is_template: - self._content = render_template_string(content, - filename=self.name, - workflow_snakecase_name=utils.to_snake_case(data.get('workflow_name', '')), - workflow_kebabcase_name=utils.to_kebab_case(data.get('workflow_name', '')), - **data) + '\n' - else: - self._content = content + if is_template: + jinja_env = Environment(loader=FileSystemLoader("/", followlinks=True), autoescape=select_autoescape()) + template = jinja_env.get_template(self.template_file_path.lstrip('/')) + self._content = template.render(filename=self.name, + workflow_snakecase_name=utils.to_snake_case(data.get('workflow_name', '')), + workflow_kebabcase_name=utils.to_kebab_case(data.get('workflow_name', '')), + **data) + '\n' + else: + with open(self.template_file_path, 'rb' if binary_mode and not is_template else 'r') as f: + content = f.read() + self._content = content return self._content def write(self, binary_mode: bool = False, output_file_path: str = None, **kwargs): diff --git a/lifemonitor/api/models/repositories/files/workflows/__init__.py b/lifemonitor/api/models/repositories/files/workflows/__init__.py index 8b37f966d..89a0be0b5 100644 --- a/lifemonitor/api/models/repositories/files/workflows/__init__.py +++ b/lifemonitor/api/models/repositories/files/workflows/__init__.py @@ -35,10 +35,10 @@ class WorkflowFile(RepositoryFile): - __workflow_types__: Dict[str, Type] = None + __workflow_types__: Dict[str, Type] | None = None - def __init__(self, repository_path: str, name: str, type: str = None, dir: str = ".", - content=None, raw_file: RepositoryFile = None) -> None: + def __init__(self, repository_path: str, name: str, type: str | None = None, dir: str = ".", + content=None, raw_file: RepositoryFile | None = None) -> None: super().__init__(repository_path, name, type, dir, content) self._raw_file = raw_file @@ -53,11 +53,11 @@ def get_content(self, binary_mode: bool = False): return super().get_content(binary_mode=binary_mode) @property - def raw_file(self) -> RepositoryFile: + def raw_file(self) -> RepositoryFile | None: return self._raw_file @classmethod - def get_workflow_extensions(cls, workflow_type: str) -> Set[str]: + def get_workflow_extensions(cls, workflow_type: str) -> Set[str] | None: try: return {_[1] for _ in cls.__get_workflow_types__()[workflow_type].__get_file_patterns__()} except AttributeError: @@ -78,13 +78,13 @@ def __from_file__(cls, file: RepositoryFile) -> WorkflowFile: dir=file.dir, content=file._content, raw_file=file) @classmethod - def __get_file_patterns__(cls, subtype: Type = None) -> Tuple[Tuple[str, str, str]]: + def __get_file_patterns__(cls, subtype: Type = None) -> Tuple[Tuple[str, str, str]] | None: return getattr(subtype or cls, "FILE_PATTERNS", None) @classmethod - def is_workflow(cls, file: RepositoryFile) -> WorkflowFile: + def is_workflow(cls, file: RepositoryFile) -> WorkflowFile | None: if not file: - return False + return None subtypes = (cls,) if cls == WorkflowFile: @@ -104,7 +104,7 @@ def is_workflow(cls, file: RepositoryFile) -> WorkflowFile: if p_dir and p_dir != f_dir: continue return subtype.__from_file__(file) - return False + return None @classmethod def __get_workflow_types__(cls) -> Dict[str, Type]: diff --git a/lifemonitor/api/models/repositories/local.py b/lifemonitor/api/models/repositories/local.py index 0a58e19cb..f9345cab9 100644 --- a/lifemonitor/api/models/repositories/local.py +++ b/lifemonitor/api/models/repositories/local.py @@ -37,7 +37,7 @@ from lifemonitor.api.models.repositories.files import (RepositoryFile, WorkflowFile) from lifemonitor.config import BaseConfig -from lifemonitor.exceptions import (DecodeROCrateException, +from lifemonitor.exceptions import (DecodeROCrateException, LifeMonitorException, NotValidROCrateException) from lifemonitor.utils import extract_zip, walk @@ -148,9 +148,14 @@ class ZippedWorkflowRepository(TemporaryLocalWorkflowRepository): def __init__(self, archive_path: str | Path, exclude: Optional[List[str]] = None, auto_cleanup: bool = True) -> None: local_path = tempfile.mkdtemp(dir=BaseConfig.BASE_TEMP_FOLDER) super().__init__(local_path=local_path, exclude=exclude, auto_cleanup=auto_cleanup) - extract_zip(archive_path, local_path) - self.archive_path = archive_path - logger.debug("Local path: %r", self.local_path) + try: + extract_zip(archive_path, local_path) + self.archive_path = archive_path + logger.debug("Local path: %r", self.local_path) + except FileNotFoundError as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + raise LifeMonitorException('Unable to process the Workflow ROCrate locally', detail=str(e), status=404) class Base64WorkflowRepository(TemporaryLocalWorkflowRepository): diff --git a/lifemonitor/api/models/rocrate/__init__.py b/lifemonitor/api/models/rocrate/__init__.py index 028ce9793..cb4863895 100644 --- a/lifemonitor/api/models/rocrate/__init__.py +++ b/lifemonitor/api/models/rocrate/__init__.py @@ -155,22 +155,21 @@ def repository(self) -> repositories.WorkflowRepository: ref = None if not os.path.exists(self.local_path): logger.debug(f"{self.local_path} archive of {self} not found locally!!!") - + logger.debug("Remote storage enabled: %r", self._storage.enabled) + logger.debug("File exists on remote storage: %r", self._storage.exists(self._get_storage_path(self.local_path))) # download the workflow ROCrate and store it into the remote storage if not inspect(self).persistent or not self._storage.exists(self._get_storage_path(self.local_path)): - # if not self._storage.enabled or not self._storage.exists(self._get_storage_path(self.local_path)): - try: - _, ref, _ = self.download_from_source(self.local_path) - logger.debug(f"RO-Crate downloaded from {self.uri} to {self.storage_path}!") + _, ref, _ = self.download_from_source(self.local_path) + logger.debug(f"RO-Crate downloaded from {self.uri} to {self.storage_path}!") + if self._storage.enabled and not self._storage.exists(self._get_storage_path(self.local_path)): self._storage.put_file_as_job(self.local_path, self._get_storage_path(self.local_path)) logger.debug(f"Scheduled job to store {self.storage_path} into the remote storage!") - except Exception as e: - logger.exception(e) else: # download the RO-Crate archive from the remote storage logger.warning(f"Getting path {self.storage_path} from remote storage!!!") - self._storage.get_file(self._get_storage_path(self.local_path), self.local_path) - logger.warning(f"Getting path {self.storage_path} from remote storage.... DONE!!!") + if self._storage.enabled and not self._storage.exists(self._get_storage_path(self.local_path)): + self._storage.get_file(self._get_storage_path(self.local_path), self.local_path) + logger.warning(f"Getting path {self.storage_path} from remote storage.... DONE!!!") # instantiate a local ROCrate repository if self._is_github_crate_(self.uri): @@ -199,33 +198,60 @@ def repository(self) -> repositories.WorkflowRepository: @property def authors(self) -> List[Dict]: - return self._crate_reader.get_authors() + return self.get_authors() + + def __get_attribute_from_crate_reader__(self, + attributeName: str, attributedType: str = 'method', + ignore_errors: bool = True, + *args, **kwargs) -> object | None: + try: + attr = getattr(self._crate_reader, attributeName) + if attributedType == 'method': + return attr(*args, **kwargs) + else: + return attr + except lm_exceptions.NotValidROCrateException as e: + logger.warning("Unable to process ROCrate archive: %s", str(e)) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + if not ignore_errors: + raise e + except Exception as e: + logger.error(str(e)) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + if not ignore_errors: + raise e + return None - def get_authors(self, suite_id: str = None) -> List[Dict]: - return self._crate_reader.get_authors(suite_id=suite_id) + def get_authors(self, suite_id: str = None) -> List[Dict] | None: + return self.__get_attribute_from_crate_reader__('get_authors', suite_id=suite_id) @hybrid_property def roc_suites(self): - return self._crate_reader.get_roc_suites() + return self.__get_attribute_from_crate_reader__('get_roc_suites') + + def get_roc_suites(self, ignore_errors: bool = False): + return self.__get_attribute_from_crate_reader__('get_roc_suites', ignore_errors=ignore_errors) - def get_roc_suite(self, roc_suite_identifier): - return self._crate_reader.get_get_roc_suite(roc_suite_identifier) + def get_roc_suite(self, roc_suite_identifier, ignore_errors: bool = False): + return self.__get_attribute_from_crate_reader__('get_get_roc_suite', roc_suite_identifier, ignore_errors=ignore_errors) @property - def based_on(self) -> str: - return self._crate_reader.isBasedOn + def based_on(self) -> str | None: + return self.__get_attribute_from_crate_reader__('isBasedOn', attributedType='property') @property def based_on_link(self) -> str: - return self._crate_reader.isBasedOn + return self.__get_attribute_from_crate_reader__('isBasedOn', attributedType='property') @property def dataset_name(self): - return self._crate_reader.dataset_name + return self.__get_attribute_from_crate_reader__('dataset_name', attributedType='property') @property def main_entity_name(self): - return self._crate_reader.main_entity_name + return self.__get_attribute_from_crate_reader__('main_entity_name', attributedType='property') def _get_authorizations(self, extra_auth: ExternalServiceAuthorizationHeader = None): authorizations = [] diff --git a/lifemonitor/api/models/services/github.py b/lifemonitor/api/models/services/github.py index d5132c13c..96a2f2830 100644 --- a/lifemonitor/api/models/services/github.py +++ b/lifemonitor/api/models/services/github.py @@ -20,10 +20,9 @@ from __future__ import annotations -import itertools import logging import re -from typing import Generator, List, Optional, Tuple +from typing import List, Optional, Tuple from urllib.error import URLError from urllib.parse import urlparse @@ -164,7 +163,8 @@ def __get_gh_workflow_runs__(self, workflow: github.Worflow.Workflow, branch=github.GithubObject.NotSet, status=github.GithubObject.NotSet, - created=github.GithubObject.NotSet): + created=github.GithubObject.NotSet, + limit: Optional[int] = None) -> CachedPaginatedList: """ Extends `Workflow.get_runs` to support `created` param """ @@ -183,16 +183,22 @@ def __get_gh_workflow_runs__(self, url_parameters["created"] = created if status is not github.GithubObject.NotSet: url_parameters["status"] = status - logger.debug("Getting runs of workflow %r ... DONE", workflow) - # return github.PaginatedList.PaginatedList( + logger.debug("Getting runs of workflow %r - branch: %r", workflow, branch) + logger.debug("Getting runs of workflow %r - status: %r", workflow, status) + logger.debug("Getting runs of workflow %r - created: %r", workflow, created) + logger.debug("Getting runs of workflow %r - params: %r", workflow, url_parameters) + # return github.PaginatedList.PaginatedList( # Default pagination class return CachedPaginatedList( github.WorkflowRun.WorkflowRun, workflow._requester, f"{workflow.url}/runs", url_parameters, None, + transactional_update=True, list_item="workflow_runs", - force_use_cache=lambda r: r.status == GithubTestingService.GithubStatus.COMPLETED + limit=limit + # disable force_use_cache: a run might be updated with new attempts even when its status is completed + # force_use_cache=lambda r: r.status == GithubTestingService.GithubStatus.COMPLETED and r.raw_data['run'] ) @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True, @@ -208,19 +214,20 @@ def __get_gh_workflow_run_attempt__(self, @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True) def __get_gh_workflow_run_attempts__(self, workflow_run: github.WorkflowRun.WorkflowRun, - limit: int = 10) -> List[github.WorkflowRun.WorkflowRun]: + limit: Optional[int] = None) -> List[github.WorkflowRun.WorkflowRun]: result = [] i = workflow_run.raw_data['run_attempt'] while i >= 1: headers, data = self.__get_gh_workflow_run_attempt__(workflow_run, i) result.append(WorkflowRun(workflow_run._requester, headers, data, True)) i -= 1 - if len(result) == limit: + if limit and len(result) == limit: break return result @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True) - def _get_gh_workflow_runs(self, workflow: Workflow.Workflow, test_instance: models.TestInstance, limit: int = 10) -> List: + def __get_workflow_runs_iterator(self, workflow: Workflow.Workflow, test_instance: models.TestInstance, + limit: Optional[int] = None) -> CachedPaginatedList: branch = github.GithubObject.NotSet created = github.GithubObject.NotSet try: @@ -228,35 +235,46 @@ def _get_gh_workflow_runs(self, workflow: Workflow.Workflow, test_instance: mode assert branch, "Branch cannot be empty" except Exception: branch = github.GithubObject.NotSet - logger.warning("No revision associated with workflow version %r", workflow) + logger.debug("No revision associated with workflow version %r", workflow) workflow_version = test_instance.test_suite.workflow_version - logger.warning("Checking Workflow version: %r (previous: %r, next: %r)", - workflow_version, workflow_version.previous_version, workflow_version.next_version) - if not workflow_version.previous_version: - if not workflow_version.next_version: - logger.debug("No previous version found, then no filter applied... Loading all available builds") - else: - created = "<{}".format(workflow_version.next_version.created.isoformat()) - elif not workflow_version.next_version: - created = ">={}".format(workflow_version.created.isoformat()) - else: + logger.debug("Checking Workflow version: %r (previous: %r, next: %r)", + workflow_version, workflow_version.previous_version, workflow_version.next_version) + if workflow_version.previous_version and workflow_version.next_version: created = "{}..{}".format(workflow_version.created.isoformat(), workflow_version.next_version.created.isoformat()) + elif workflow_version.previous_version: + created = ">={}".format(workflow_version.created.isoformat()) + elif workflow_version.next_version: + created = "<{}".format(workflow_version.next_version.created.isoformat()) + else: + logger.debug("No previous version found, then no filter applied... Loading all available builds") logger.debug("Fetching runs : %r - %r", branch, created) # return list(self.__get_gh_workflow_runs__(workflow, branch=branch, created=created)) - return list(itertools.islice(self.__get_gh_workflow_runs__(workflow, branch=branch, created=created), limit)) + # return list(itertools.islice(self.__get_gh_workflow_runs__(workflow, branch=branch, created=created), limit)) + return self.__get_gh_workflow_runs__(workflow, branch=branch, created=created, limit=limit) @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True) def _list_workflow_runs(self, test_instance: models.TestInstance, - status: str = None, limit: int = 10) -> Generator[github.WorkflowRun.WorkflowRun]: + status: Optional[str] = None, limit: int = 10) -> List[github.WorkflowRun.WorkflowRun]: # get gh workflow workflow = self._get_gh_workflow_from_test_instance_resource(test_instance.resource) logger.debug("Retrieved workflow %s from github", workflow) - logger.warning("Workflow Runs Limit: %r", limit) - logger.warning("Workflow Runs Status: %r", status) + logger.debug("Workflow Runs Limit: %r", limit) + logger.debug("Workflow Runs Status: %r", status) + + return list(self.__get_workflow_runs_iterator(workflow, test_instance, limit=limit)) + + @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True) + def _list_workflow_run_attempts(self, test_instance: models.TestInstance, + status: Optional[str] = None, limit: int = 10) -> List[github.WorkflowRun.WorkflowRun]: + # get gh workflow + workflow = self._get_gh_workflow_from_test_instance_resource(test_instance.resource) + logger.debug("Retrieved workflow %s from github", workflow) + logger.debug("Workflow Runs Limit: %r", limit) + logger.debug("Workflow Runs Status: %r", status) result = [] - for run in self._get_gh_workflow_runs(workflow, test_instance, limit=limit): + for run in self.__get_workflow_runs_iterator(workflow, test_instance): logger.debug("Loading Github run ID %r", run.id) # The Workflow.get_runs method in the PyGithub API has a status argument # which in theory we could use to filter the runs that are retrieved to @@ -269,15 +287,18 @@ def _list_workflow_runs(self, test_instance: models.TestInstance, # if status is None or run.status == status: logger.debug("Number of attempts of run ID %r: %r", run.id, run.raw_data['run_attempt']) if (limit is None or limit > 1) and run.raw_data['run_attempt'] > 1: - for attempt in self.__get_gh_workflow_run_attempts__(run, limit=limit - len(result)): + for attempt in self.__get_gh_workflow_run_attempts__( + run, limit=(limit - len(result) if limit else None)): + logger.debug("Attempt: %r %r %r", attempt, status, attempt.status) if status is None or attempt.status == status: result.append(attempt) else: if status is None or run.status == status: result.append(run) - if limit and len(result) == limit: + # stop iteration if the limit is reached + if len(result) >= limit: break - result = result if len(result) == 1 else sorted(result, key=lambda x: x.updated_at, reverse=True)[:limit] + for run in result: logger.debug("Run: %r --> %r -- %r", run, run.created_at, run.updated_at) return result @@ -285,7 +306,7 @@ def _list_workflow_runs(self, test_instance: models.TestInstance, def get_last_test_build(self, test_instance: models.TestInstance) -> Optional[GithubTestBuild]: try: logger.debug("Getting latest build...") - for run in self._list_workflow_runs(test_instance, status=self.GithubStatus.COMPLETED): + for run in self._list_workflow_run_attempts(test_instance, status=self.GithubStatus.COMPLETED): return GithubTestBuild(self, test_instance, run) logger.debug("Getting latest build... DONE") return None @@ -295,7 +316,7 @@ def get_last_test_build(self, test_instance: models.TestInstance) -> Optional[Gi def get_last_passed_test_build(self, test_instance: models.TestInstance) -> Optional[GithubTestBuild]: try: logger.debug("Getting last passed build...") - for run in self._list_workflow_runs(test_instance, status=self.GithubStatus.COMPLETED): + for run in self._list_workflow_run_attempts(test_instance, status=self.GithubStatus.COMPLETED): if run.conclusion == self.GithubConclusion.SUCCESS: return GithubTestBuild(self, test_instance, run) return None @@ -305,7 +326,7 @@ def get_last_passed_test_build(self, test_instance: models.TestInstance) -> Opti def get_last_failed_test_build(self, test_instance: models.TestInstance) -> Optional[GithubTestBuild]: try: logger.debug("Getting last failed build...") - for run in self._list_workflow_runs(test_instance, status=self.GithubStatus.COMPLETED): + for run in self._list_workflow_run_attempts(test_instance, status=self.GithubStatus.COMPLETED): if run.conclusion == self.GithubConclusion.FAILURE: return GithubTestBuild(self, test_instance, run) return None @@ -316,7 +337,7 @@ def get_test_builds(self, test_instance: models.TestInstance, limit=10) -> list: try: logger.debug("Getting test builds...") return [GithubTestBuild(self, test_instance, run) - for run in self._list_workflow_runs(test_instance, limit=limit)[:limit]] + for run in self._list_workflow_run_attempts(test_instance, limit=limit)[:limit]] except GithubRateLimitExceededException as e: raise lm_exceptions.RateLimitExceededException(detail=str(e), instance=test_instance) @@ -365,9 +386,10 @@ def get_test_build_external_link(self, test_build: models.TestBuild) -> str: def get_test_build_output(self, test_instance: models.TestInstance, build_number, offset_bytes=0, limit_bytes=131072): raise lm_exceptions.NotImplementedException(detail="not supported for GitHub test builds") - def start_test_build(self, test_instance: models.TestInstance) -> bool: + def start_test_build(self, test_instance: models.TestInstance, build_number: int = None) -> bool: try: - last_build = self.get_last_test_build(test_instance) + last_build = self.get_last_test_build(test_instance) \ + if build_number is None else self.get_test_build(test_instance, build_number) assert last_build if last_build: run: WorkflowRun = last_build._metadata diff --git a/lifemonitor/api/models/status.py b/lifemonitor/api/models/status.py index 4f1212fff..f401b9055 100644 --- a/lifemonitor/api/models/status.py +++ b/lifemonitor/api/models/status.py @@ -93,6 +93,7 @@ def check_status(cls, suites): }) for test_instance in suite.test_instances: try: + latest_test_builds = test_instance.get_test_builds(limit=10) latest_build = test_instance.last_test_build if latest_build is None: availability_issues.append({ @@ -102,7 +103,7 @@ def check_status(cls, suites): }) else: # Search the latest completed build - for latest_build in test_instance.get_test_builds(): + for latest_build in latest_test_builds: logger.debug("Checking build %r: %r", latest_build, latest_build.status) if not cls._skip_build(latest_build): break diff --git a/lifemonitor/api/models/testsuites/testinstance.py b/lifemonitor/api/models/testsuites/testinstance.py index 90c354ec8..990946050 100644 --- a/lifemonitor/api/models/testsuites/testinstance.py +++ b/lifemonitor/api/models/testsuites/testinstance.py @@ -20,6 +20,7 @@ from __future__ import annotations +import datetime import logging import uuid as _uuid from typing import List @@ -46,6 +47,8 @@ class TestInstance(db.Model, ModelMixin): parameters = db.Column(JSON, nullable=True) submitter_id = db.Column(db.Integer, db.ForeignKey(models.User.id), nullable=True) + last_builds_update = db.Column(db.DateTime, default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow) # configure relationships submitter = db.relationship("User", uselist=False) test_suite = db.relationship("TestSuite", @@ -115,12 +118,18 @@ def get_last_test_build(self): @cached(timeout=Timeout.NONE, client_scope=False, transactional_update=True) def get_test_builds(self, limit=10): - return self.testing_service.get_test_builds(self, limit=limit) + try: + return self.testing_service.get_test_builds(self, limit=limit) + finally: + self.last_builds_updated() @cached(timeout=Timeout.BUILD, client_scope=False, transactional_update=True) def get_test_build(self, build_number): return self.testing_service.get_test_build(self, build_number) + def last_builds_updated(self, when=datetime.datetime.utcnow()): + self.last_builds_update = when + def to_dict(self, test_build=False, test_output=False): data = { 'uuid': str(self.uuid), diff --git a/lifemonitor/api/models/wizards/__init__.py b/lifemonitor/api/models/wizards/__init__.py index 279047eab..e693f8866 100644 --- a/lifemonitor/api/models/wizards/__init__.py +++ b/lifemonitor/api/models/wizards/__init__.py @@ -29,7 +29,7 @@ from hashlib import sha1 from importlib import import_module from posixpath import basename, dirname -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Type from genericpath import isfile from lifemonitor.api.models.repositories.base import WorkflowRepository @@ -44,22 +44,22 @@ class Wizard(): # map issue -> wizard - __wizards__: Dict = None + __wizards__: Optional[Dict] = None # main wizard attributes title: str = "Wizard" description: str = "" steps: List[Step] = [] - issue: WorkflowRepositoryIssue = None + issue: Optional[Type] = None def __init__(self, issue: WorkflowRepositoryIssue, - steps: List[Step] = None, - io_handler: IOHandler = None): + steps: Optional[List[Step]] = None, + io_handler: Optional[IOHandler] = None): self.issue = issue self._io_handler = io_handler - self.current_step: Step = None + self.current_step: Optional[Step] = None self._steps_list = steps if steps else self.steps - self.__steps: List[Step] = None + self.__steps: Optional[List[Step]] = None @property def id(self) -> str: @@ -76,7 +76,7 @@ def _steps(self) -> List[Step]: return self.__steps @property - def io_handler(self) -> IOHandler: + def io_handler(self) -> Optional[IOHandler]: return self._io_handler @io_handler.setter diff --git a/lifemonitor/api/models/workflows.py b/lifemonitor/api/models/workflows.py index deed1e062..504c537f1 100644 --- a/lifemonitor/api/models/workflows.py +++ b/lifemonitor/api/models/workflows.py @@ -21,7 +21,7 @@ from __future__ import annotations import logging -from typing import List, Set, Union +from typing import List, Optional, Set, Union import lifemonitor.api.models as models import lifemonitor.exceptions as lm_exceptions @@ -91,7 +91,7 @@ def get_registry_identifier(self, registry: WorkflowRegistry) -> str: @hybrid_property def latest_version(self) -> WorkflowVersion: - return max(self.versions.values(), key=lambda v: v.modified) + return max(self.versions.values(), key=lambda v: v.created) def add_version(self, version, uri, submitter: User, uuid=None, name=None): return WorkflowVersion(self, uri, version, submitter, uuid=uuid, name=name) @@ -205,6 +205,7 @@ class WorkflowVersion(ROCrate): db.Column(db.Integer, db.ForeignKey("workflow.id"), nullable=False) workflow = db.relationship("Workflow", foreign_keys=[workflow_id], cascade="all", backref=db.backref("versions", cascade="all, delete-orphan", + order_by="desc(WorkflowVersion.created)", collection_class=attribute_mapped_collection('version'))) test_suites = db.relationship("TestSuite", back_populates="workflow_version", cascade="all, delete") @@ -232,24 +233,28 @@ def __repr__(self): def _storage(self) -> RemoteStorage: return RemoteStorage() + def _get_relative_version(self, delta_index=1) -> Optional[WorkflowVersion]: + try: + values = list(self.workflow.versions.values()) + self_index = values.index(self) + index = self_index + delta_index + if index >= 0 and index < len(values): + return values[self_index + delta_index] + except ValueError: + message = f"{self} doesn't belong to the workflow {self.workflow}" + logger.error(message) + raise lm_exceptions.LifeMonitorException('Value error', detail=message) + except IndexError: + pass + return None + @property def previous_version(self) -> WorkflowVersion: - previous = None - for v in self.workflow.versions.values(): - if v == self: - return previous - previous = v - return None + return self._get_relative_version(delta_index=1) @property def next_version(self) -> WorkflowVersion: - found = False - for v in self.workflow.versions.values(): - if v == self: - found = True - elif found: - return v - return None + return self._get_relative_version(delta_index=-1) def check_health(self) -> dict: health = {'healthy': True, 'issues': []} @@ -301,11 +306,11 @@ def is_latest(self) -> bool: @property def previous_versions(self) -> List[str]: - return [w.version for w in self.workflow.versions.values() if w != self and w.version < self.version] + return [w.version for w in self.workflow.versions.values() if w != self and w.created < self.created] @property def previous_workflow_versions(self) -> List[models.WorkflowVersion]: - return [w for w in self.workflow.versions.values() if w != self and w.version < self.version] + return [w for w in self.workflow.versions.values() if w != self and w.created < self.created] @property def status(self) -> models.WorkflowStatus: @@ -340,6 +345,11 @@ def to_dict(self, test_suite=False, test_build=False, test_output=False): for s in self.test_suites] return data + def save(self): + self.workflow.save(commit=False, flush=False) + self.modified = self.workflow.modified + super().save(update_modified=False) + def delete(self): if len(self.workflow.versions) > 1: workflow = self.workflow diff --git a/lifemonitor/api/serializers.py b/lifemonitor/api/serializers.py index 2f67e1e99..218dc4e46 100644 --- a/lifemonitor/api/serializers.py +++ b/lifemonitor/api/serializers.py @@ -188,15 +188,19 @@ def get_rocrate(self, obj: models.WorkflowVersion): f"workflows/{obj.workflow.uuid}/rocrate/{obj.version}/download") } } - if obj.based_on: - rocrate['links']['based_on'] = obj.based_on - rocrate['links']['registries'] = {} - for r_name, rv in obj.registry_workflow_versions.items(): - rocrate['links']['registries'][r_name] = rv.link - rocrate['metadata'] = obj.crate_metadata - if 'rocrate_metadata' in self.exclude or \ - self.only and 'rocrate_metadata' not in self.only: - del rocrate['metadata'] + try: + if obj.based_on: + rocrate['links']['based_on'] = obj.based_on + rocrate['links']['registries'] = {} + for r_name, rv in obj.registry_workflow_versions.items(): + rocrate['links']['registries'][r_name] = rv.link + rocrate['metadata'] = obj.crate_metadata + if 'rocrate_metadata' in self.exclude or \ + self.only and 'rocrate_metadata' not in self.only: + del rocrate['metadata'] + except Exception as e: + rocrate['unavailable'] = True + rocrate['unavailability_reason'] = str(e) return rocrate @post_dump @@ -452,7 +456,7 @@ def get_status(self, workflow): try: result = { "aggregate_test_status": workflow.latest_version.status.aggregated_status, - "latest_build": self.get_latest_build(workflow) + "latest_builds": self.get_latest_builds(workflow) } reason = format_availability_issues(workflow.latest_version.status) if reason: @@ -488,12 +492,13 @@ def get_versions(self, workflow): logger.exception(e) return None - def get_latest_build(self, workflow): + def get_latest_builds(self, workflow): try: latest_builds = workflow.latest_version.status.latest_builds + builds = [] if latest_builds and len(latest_builds) > 0: - return BuildSummarySchema(exclude=('meta', 'links')).dump(latest_builds[0]) - return None + builds.append(BuildSummarySchema(exclude=('meta', 'links')).dump(latest_builds[0])) + return builds except Exception as e: logger.debug(e) return None @@ -553,7 +558,7 @@ class Meta: definition = fields.Method("get_definition") instances = fields.Nested(TestInstanceSchema(self_link=False, exclude=('meta',)), attribute="test_instances", many=True) - status = fields.Method("get_status") + aggregate_test_status = fields.Method("get_status") latest_builds = fields.Method("get_latest_builds") def __init__(self, *args, self_link: bool = True, @@ -568,9 +573,11 @@ def get_definition(self, obj): def get_status(self, obj): try: - return SuiteStatusSchema(only=('status',)).dump(obj)['status'] if self.status else None - except Exception: + return SuiteStatusSchema(only=('aggregate_test_status',)).dump(obj)['aggregate_test_status'] if self.status else None + except Exception as e: logger.warning("Unable to extract status for suite: %r", obj) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) return None def get_latest_builds(self, obj): @@ -602,7 +609,7 @@ def __init__(self, *args, def get_items(self, obj): exclude = ['meta', 'links'] if not self.status: - exclude.append('status') + exclude.append('aggregate_test_status') if not self.latest_builds: exclude.append('latest_builds') return [self.__item_scheme__( @@ -620,7 +627,7 @@ class Meta: model = models.TestSuite suite_uuid = fields.String(attribute="uuid") - status = fields.Method("get_aggregated_status") + aggregate_test_status = fields.Method("get_aggregate_status") latest_builds = fields.Method("get_builds") reason = fields.Method("get_reason") _errors = [] @@ -642,7 +649,7 @@ def get_reason(self, suite): except Exception as e: return str(e) - def get_aggregated_status(self, suite): + def get_aggregate_status(self, suite): try: return suite.status.aggregated_status except Exception as e: diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 2bcbcd566..09ba086c7 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -21,8 +21,8 @@ from __future__ import annotations import logging -from datetime import datetime -from typing import List, Optional, Union +from datetime import datetime, timezone +from typing import Dict, List, Optional, Union import lifemonitor.exceptions as lm_exceptions from lifemonitor.api import models @@ -33,7 +33,9 @@ from lifemonitor.auth.oauth2.client import providers from lifemonitor.auth.oauth2.client.models import OAuthIdentity from lifemonitor.auth.oauth2.server import server +from lifemonitor.tasks.models import Job from lifemonitor.utils import OpenApiSpecs, ROCrateLinkContext, to_snake_case +from lifemonitor.ws import io logger = logging.getLogger() @@ -110,7 +112,7 @@ def _find_and_check_workflow_version(cls, user: User, uuid, version=None): def register_workflow(cls, rocrate_or_link, workflow_submitter: User, workflow_version, workflow_uuid=None, workflow_identifier=None, workflow_registry: Optional[models.WorkflowRegistry] = None, - authorization=None, name=None, public=False): + authorization=None, name=None, public=False, job: Job = None): # reference to the workflow w = None @@ -150,13 +152,19 @@ def register_workflow(cls, rocrate_or_link, workflow_submitter: User, workflow_v if not workflow_registry: raise ValueError("Missing ROC link") else: + if job: + job.update_status('Getting workflow info from registry', save=True) roc_link = workflow_registry.get_rocrate_external_link(workflow_identifier, workflow_version) # register workflow version + if job: + job.update_status('Adding workflow version', save=True) wv = w.add_version(workflow_version, roc_link, workflow_submitter, name=name) # associate workflow version to the workflow registry if workflow_registry: + if job: + job.update_status('Associating workflow version to registry', save=True) workflow_registry.add_workflow_version(wv, workflow_identifier, workflow_version) # set permissions @@ -191,7 +199,9 @@ def register_workflow(cls, rocrate_or_link, workflow_submitter: User, workflow_v # parse roc_metadata and register suites and instances try: - if wv.roc_suites: + if job: + job.update_status('Processing test suites', save=True) + if wv.get_roc_suites(): for _, raw_suite in wv.roc_suites.items(): cls._init_test_suite_from_json(wv, workflow_submitter, raw_suite) except KeyError as e: @@ -277,6 +287,18 @@ def update_workflow(cls, workflow_submitter: User, w.public = public # store the new version w.save() + + try: + logger.debug("Notifying workflow version update...") + io.publish_message({ + "type": "sync", + "data": [{'uuid': w.uuid, "version": workflow_version, 'lastUpdate': w.modified.timestamp()}] + }) + logger.debug("Notifying workflow version update... DONE") + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + logger.error(str(e)) return wv @classmethod @@ -528,7 +550,13 @@ def get_public_workflows() -> List[models.Workflow]: return models.Workflow.get_public_workflows() @staticmethod - def get_user_workflows(user: User, include_subscriptions: bool = False) -> List[models.Workflow]: + def get_user_workflows(user: User, + include_subscriptions: bool = False, + only_subscriptions: bool = False) -> List[models.Workflow]: + if only_subscriptions: + return [_.resource for _ in user.subscriptions + if _.resource.public or user in [p.user for p in _.resource.permissions]] + workflows = [w for w in models.Workflow.get_user_workflows(user, include_subscriptions=include_subscriptions)] for svc in models.WorkflowRegistry.all(): if svc.get_user(user.id): @@ -578,6 +606,10 @@ def get_suite(suite_uuid) -> models.TestSuite: def get_test_instance(instance_uuid) -> models.TestInstance: return models.TestInstance.find_by_uuid(instance_uuid) + @staticmethod + def get_user_by_id(user_id: str) -> User: + return User.find_by_id(user_id) + @staticmethod def find_registry_user_identity(registry: models.WorkflowRegistry, internal_id=None, external_id=None) -> OAuthIdentity: @@ -682,3 +714,39 @@ def deleteUserNotifications(user: User, list_of_uuids: List[str]): return lm_exceptions.EntityNotFoundException(Notification, entity_id=n_uuid) user.notifications.remove(n) user.save() + + @staticmethod + def list_workflow_updates(since: datetime = datetime.now) -> Dict[str, datetime]: + from lifemonitor.db import db + query = """ + SELECT DISTINCT uuid, version, max(last_update) as last_update + FROM ( + SELECT w.uuid AS uuid, r.version AS version, GREATEST(r.modified, w.modified) AS last_update + FROM resource AS r + JOIN workflow_version AS wv ON r.id = wv.id + JOIN resource AS w ON wv.workflow_id = w.id + WHERE r.type LIKE 'workflow_version' + + UNION + + SELECT w.uuid AS uuid, r.version AS version, t.last_builds_update AS last_update + FROM test_instance AS t + JOIN test_suite AS s ON t.test_suite_uuid = s.uuid + JOIN workflow_version AS wv ON s.workflow_version_id = wv.id + JOIN resource AS r ON r.id = wv.id + JOIN resource AS w ON wv.workflow_id = w.id + ) as X + GROUP BY X.uuid,X.version + ORDER BY last_update DESC + """ + result: List = [] + rs = db.session.execute(query) + for row in rs: + logger.debug("Row: %r", row) + result.append({ + "uuid": str(row[0]), + "version": row[1], + "lastUpdate": row[2].replace(tzinfo=timezone.utc).timestamp() # time.mktime(row[2].timetuple()) + # "lastUpdate": int(row[2].timestamp() * 1000) + }) + return result diff --git a/lifemonitor/app.py b/lifemonitor/app.py index aa8270e37..d37a60628 100644 --- a/lifemonitor/app.py +++ b/lifemonitor/app.py @@ -22,13 +22,15 @@ import os import time -from flask import Flask, jsonify, redirect, request +from flask import Flask, jsonify, redirect, render_template, request, url_for from flask_cors import CORS from flask_migrate import Migrate +from lifemonitor import redis import lifemonitor.config as config -from lifemonitor import __version__ as version +from lifemonitor.auth.services import current_user from lifemonitor.integrations import init_integrations +from lifemonitor.metrics import init_metrics from lifemonitor.routes import register_routes from lifemonitor.tasks import init_task_queues @@ -63,8 +65,6 @@ def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=T flask_app_instance_path = getattr(app_config, "FLASK_APP_INSTANCE_PATH", None) # create Flask app instance app = Flask(__name__, instance_relative_config=True, instance_path=flask_app_instance_path, **kwargs) - # enable CORS - CORS(app) # register handler for app specific exception app.register_error_handler(Exception, handle_exception) # set config object @@ -83,6 +83,16 @@ def create_app(env=None, settings=None, init_app=True, worker=False, load_jobs=T with app.app_context() as ctx: initialize_app(app, ctx, load_jobs=load_jobs) + @app.route("/") + def index(): + if not current_user.is_authenticated: + return render_template("index.j2") + return redirect(url_for('auth.index')) + + @app.route("/profile") + def profile(): + return redirect(url_for('auth.index', back=request.args.get('back', False))) + # append routes to check app health @app.route("/health") def health(): @@ -120,8 +130,12 @@ def log_response(response): def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True): # init tmp folder os.makedirs(app.config.get('BASE_TEMP_FOLDER'), exist_ok=True) + # enable CORS + CORS(app, expose_headers=["Content-Type", "X-CSRFToken"], supports_credentials=True) # configure logging config.configure_logging(app) + # init Redis connection + redis.init(app) # configure app DB db.init_app(app) # initialize Migration engine @@ -138,24 +152,7 @@ def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool init_mail(app) # initialize integrations init_integrations(app) + # initialize metrics engine + init_metrics(app, prom_registry) # register commands commands.register_commands(app) - - # configure prometheus exporter - # must be configured after the routes are registered - metrics_class = None - if os.environ.get('FLASK_ENV') == 'production': - if 'PROMETHEUS_MULTIPROC_DIR' in os.environ: - from prometheus_flask_exporter.multiprocess import \ - GunicornPrometheusMetrics - metrics_class = GunicornPrometheusMetrics - else: - logger.warning("Unable to start multiprocess prometheus exporter: 'PROMETHEUS_MULTIPROC_DIR' not set." - "Metrics will be exposed through the `/metrics` endpoint.") - if not metrics_class: - from prometheus_flask_exporter import PrometheusMetrics - metrics_class = PrometheusMetrics - - metrics = metrics_class(app, defaults_prefix='lm', registry=prom_registry) - metrics.info('app_info', "LifeMonitor service", version=version) - app.metrics = metrics diff --git a/lifemonitor/auth/controllers.py b/lifemonitor/auth/controllers.py index 5ddf485d3..3ac44c569 100644 --- a/lifemonitor/auth/controllers.py +++ b/lifemonitor/auth/controllers.py @@ -44,8 +44,9 @@ logger = logging.getLogger(__name__) blueprint = flask.Blueprint("auth", __name__, + url_prefix="/account", template_folder='templates', - static_folder="static", static_url_path='/static') + static_folder="static", static_url_path='../static') # Set the login view login_manager.login_view = "auth.login" @@ -158,7 +159,7 @@ def get_registry_user(user_id): @blueprint.route("/", methods=("GET",)) def index(): - return redirect(url_for('auth.profile')) + return redirect(url_for('auth.profile', back=request.args.get('back', False))) @blueprint.route("/profile", methods=("GET",)) @@ -197,6 +198,7 @@ def register(): if flask.request.method == "GET": # properly intialize/clear the session before the registration flask.session["confirm_user_details"] = True + flask.session["sign_in"] = False save_current_user_identity(None) with db.session.no_autoflush: form = RegisterForm() @@ -208,7 +210,24 @@ def register(): clear_cache() return redirect(url_for("auth.index")) return render_template("auth/register.j2", form=form, - action='/register', providers=get_providers()) + action=url_for('auth.register'), providers=get_providers()) + + +@blueprint.route("/identity_not_found", methods=("GET", "POST")) +def identity_not_found(): + identity = get_current_user_identity() + logger.debug("Current provider identity: %r", identity) + if not identity or not identity.user: + flash("Unable to register the user") + return redirect(url_for("auth.login")) + form = RegisterForm() + user = identity.user + # workaround to force clean DB session + from lifemonitor.db import db + db.session.rollback() + return render_template("auth/identity_not_found.j2", form=form, + action=url_for('auth.register_identity') if flask.session.get('sign_in', False) else url_for('auth.register'), + identity=identity, user=user, providers=get_providers()) @blueprint.route("/register_identity", methods=("GET", "POST")) @@ -230,7 +249,7 @@ def register_identity(): flash("Account created", category="success") clear_cache() return redirect(url_for("auth.index")) - return render_template("auth/register.j2", form=form, action='/register_identity', + return render_template("auth/register.j2", form=form, action=url_for('auth.register'), identity=identity, user=user, providers=get_providers()) @@ -239,13 +258,14 @@ def register_identity(): def login(): form = LoginForm() flask.session["confirm_user_details"] = True + flask.session["sign_in"] = True if form.validate_on_submit(): user = form.get_user() if user: login_user(user) session.pop('_flashes', None) flash("You have logged in", category="success") - return redirect(NextRouteRegistry.pop(url_for("auth.index"))) + return redirect(NextRouteRegistry.pop(url_for("auth.profile"))) return render_template("auth/login.j2", form=form, providers=get_providers()) @@ -256,7 +276,7 @@ def logout(): session.pop('_flashes', None) flash("You have logged out", category="success") NextRouteRegistry.clear() - return redirect(url_for("auth.index")) + return redirect('/') @blueprint.route("/delete_account", methods=("POST",)) @@ -402,7 +422,7 @@ def enable_registry_sync(): form = RegistrySettingsForm() if form.validate_on_submit(): registry_name = request.values.get("registry", None) - return redirect(f'/oauth2/login/{registry_name}?scope=read+write&next=/enable_registry_sync?s={registry_name}') + return redirect(f'/oauth2/login/{registry_name}?scope=read+write&next=/account/enable_registry_sync?s={registry_name}') else: logger.debug("Form validation failed") flash("Invalid request", category="error") diff --git a/lifemonitor/auth/models.py b/lifemonitor/auth/models.py index a4d106398..f2149a6b4 100644 --- a/lifemonitor/auth/models.py +++ b/lifemonitor/auth/models.py @@ -277,6 +277,10 @@ def to_dict(self): } } + @classmethod + def find_by_id(cls, user_id): + return cls.query.filter(cls.id == user_id).first() + @classmethod def find_by_username(cls, username): return cls.query.filter(cls.username == username).first() diff --git a/lifemonitor/auth/oauth2/client/controllers.py b/lifemonitor/auth/oauth2/client/controllers.py index b5e76447d..1e6682808 100644 --- a/lifemonitor/auth/oauth2/client/controllers.py +++ b/lifemonitor/auth/oauth2/client/controllers.py @@ -74,11 +74,11 @@ def authorize(name): else: # handle failed return _handle_authorize(remote, None, None) - if 'id_token' in token: - user_info = remote.parse_id_token(token) - else: - remote.token = token - user_info = remote.userinfo(token=token) + # if 'id_token' in token: + # user_info = remote.parse_id_token(token) + # else: + remote.token = token + user_info = remote.userinfo(token=token) return _handle_authorize(remote, token, user_info) except OAuthError as e: logger.debug(e) @@ -94,6 +94,9 @@ def login(name, scope: str = None): remote = oauth2_registry.create_client(name) if remote is None: abort(404) + action = request.args.get('action', False) + if action and action == 'sign-in': + session['sign_in'] = True redirect_uri = url_for('.authorize', name=name, _external=True) conf_key = '{}_AUTHORIZE_PARAMS'.format(name.upper()) params = current_app.config.get(conf_key, {}) @@ -162,13 +165,26 @@ def handle_authorize(self, provider: FlaskRemoteApp, token, user_info: OAuthUser logger.debug("Update identity token: %r -> %r", identity.token, token) except OAuthIdentityNotFoundException: logger.debug("Not found OAuth identity <%r,%r>", provider.name, user_info.sub) - with db.session.no_autoflush: - identity = OAuthIdentity( - provider=p, - user_info=user_info.to_dict(), - provider_user_id=user_info.sub, - token=token, - ) + logger.debug("SignIn: %r", session.get('sign_in', False)) + # with db.session.no_autoflush: + identity = OAuthIdentity( + provider=p, + user_info=user_info.to_dict(), + provider_user_id=user_info.sub, + token=token, + ) + save_current_user_identity(identity) + try: + if session['sign_in']: + return redirect(url_for("auth.identity_not_found")) + except KeyError as e: + logger.error(e) + + try: + session.pop('sign_in', False) + except KeyError as e: + logger.debug(e) + # Now, figure out what to do with this token. There are 2x2 options: # user login state and token link state. if current_user.is_anonymous: diff --git a/lifemonitor/auth/oauth2/client/providers/github.py b/lifemonitor/auth/oauth2/client/providers/github.py index 23f0c8cd4..450464e6f 100644 --- a/lifemonitor/auth/oauth2/client/providers/github.py +++ b/lifemonitor/auth/oauth2/client/providers/github.py @@ -53,7 +53,7 @@ def normalize_userinfo(client, data): class GitHub: - name = 'Github' + name = 'GitHub' client_name = 'github' oauth_config = { 'client_id': current_app.config.get('GITHUB_CLIENT_ID', None), diff --git a/lifemonitor/auth/oauth2/client/providers/lsaai.py b/lifemonitor/auth/oauth2/client/providers/lsaai.py new file mode 100644 index 000000000..8ca4a01d8 --- /dev/null +++ b/lifemonitor/auth/oauth2/client/providers/lsaai.py @@ -0,0 +1,80 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import logging +from flask import current_app + +# Config a module level logger +logger = logging.getLogger(__name__) + + +def normalize_userinfo(client, data): + logger.debug("LSAAI Data: %r", data) + preferred_username = data.get('eduperson_principal_name')[0].replace('@lifescience-ri.eu', '') \ + if 'eduperson_principal_namex' in data and len(data['eduperson_principal_name']) > 0 \ + else data['name'].replace(' ', '') + params = { + 'sub': str(data['sub']), + 'name': data['name'], + 'email': data.get('email'), + 'preferred_username': preferred_username + # 'profile': data['html_url'], + # 'picture': data['avatar_url'], + # 'website': data.get('blog'), + } + + # The email can be be None despite the scope being 'user:email'. + # That is because a user can choose to make his/her email private. + # If that is the case we get all the users emails regardless if private or note + # and use the one he/she has marked as `primary` + try: + if params.get('email') is None: + resp = client.get('user/emails') + resp.raise_for_status() + data = resp.json() + params["email"] = next(email['email'] for email in data if email['primary']) + except Exception as e: + logger.warning("Unable to get user email. Reason: %r", str(e)) + return params + + +class LsAAI: + name = 'LifeScience RI' + client_name = 'lsaai' + oauth_config = { + 'client_id': current_app.config.get('LSAAI_CLIENT_ID', None), + 'client_secret': current_app.config.get('LSAAI_CLIENT_SECRET', None), + 'client_name': client_name, + 'uri': 'https://proxy.aai.lifescience-ri.eu', + 'api_base_url': 'https://proxy.aai.lifescience-ri.eu', + 'access_token_url': 'https://proxy.aai.lifescience-ri.eu/OIDC/token', + 'authorize_url': 'https://proxy.aai.lifescience-ri.eu/saml2sp/OIDC/authorization', + 'client_kwargs': {'scope': 'openid profile email orcid eduperson_principal_name'}, + 'userinfo_endpoint': 'https://proxy.aai.lifescience-ri.eu/OIDC/userinfo', + 'userinfo_compliance_fix': normalize_userinfo, + 'server_metadata_url': 'https://proxy.aai.lifescience-ri.eu/.well-known/openid-configuration' + } + + def __repr__(self) -> str: + return "LSAAI Provider" + + @staticmethod + def normalize_userinfo(client, data): + return normalize_userinfo(client, data) diff --git a/lifemonitor/auth/oauth2/client/services.py b/lifemonitor/auth/oauth2/client/services.py index f9882957e..e305ba274 100644 --- a/lifemonitor/auth/oauth2/client/services.py +++ b/lifemonitor/auth/oauth2/client/services.py @@ -38,6 +38,7 @@ def get_providers(skip_registration: bool = False): from .providers.github import GitHub + from .providers.lsaai import LsAAI from .providers.seek import Seek providers = [] logger.debug("Preparing list of providers...") @@ -45,6 +46,9 @@ def get_providers(skip_registration: bool = False): if current_app.config.get('GITHUB_CLIENT_ID', None) \ and current_app.config.get('GITHUB_CLIENT_SECRET', None): providers.append(GitHub) + if current_app.config.get('LSAAI_CLIENT_ID', None) \ + and current_app.config.get('LSAAI_CLIENT_SECRET', None): + providers.append(LsAAI) # set workflow registries as oauth providers if db_initialized(): try: @@ -92,7 +96,9 @@ def merge_users(merge_from: User, merge_into: User, provider: str): def save_current_user_identity(identity: OAuthIdentity): - session["oauth2_username"] = identity.user.username if identity else None + session["oauth2_username"] = identity.user.username \ + if identity and identity.user \ + else identity.user_info["preferred_username"] if identity and identity.user_info else None session["oauth2_provider_name"] = identity.provider.client_name if identity else None session["oauth2_user_info"] = identity.user_info if identity else None session["oauth2_user_token"] = identity.token if identity else None diff --git a/lifemonitor/auth/oauth2/server/templates/authorize.html b/lifemonitor/auth/oauth2/server/templates/authorize.html index 4000c07d7..0fdd8bf77 100644 --- a/lifemonitor/auth/oauth2/server/templates/authorize.html +++ b/lifemonitor/auth/oauth2/server/templates/authorize.html @@ -1,5 +1,5 @@ -{% extends 'auth/base.j2' %} -{% import 'auth/macros.j2' as macros %} +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} {% block body_class %} login-page {% endblock %} {% block body_style %} height: auto; {% endblock %} diff --git a/lifemonitor/auth/services.py b/lifemonitor/auth/services.py index 199f3cb47..753dd9cd8 100644 --- a/lifemonitor/auth/services.py +++ b/lifemonitor/auth/services.py @@ -91,23 +91,43 @@ def _current_registry(): current_user = flask_login.current_user +def is_user_or_registry_authenticated(): + logger.debug(f"The current user: {current_user}") + logger.debug(f"The current registry: {current_registry}") + logger.debug(f"Request args: {request.args}") + # raise unauthorized if no user nor registry in session + if not current_registry and current_user.is_anonymous: + raise NotAuthorizedException(detail=messages.unauthorized_no_user_nor_registry) + # if there is a registry user in session + # check whether his token issued by the registry is valid + if current_registry and not current_user.is_anonymous: + if current_registry.name not in current_user.oauth_identity: + raise NotAuthorizedException( + detail=messages.unauthorized_user_without_registry_identity.format(current_registry.name), + authorization_url=url_for('oauth2provider.authorize', + name=current_registry.name, next=request.url)) + + def authorized(func): @wraps(func) def wrapper(*args, **kwargs): - logger.debug(f"The current user: {current_user}") - logger.debug(f"The current registry: {current_registry}") - logger.debug(f"Request args: {request.args}") - # raise unauthorized if no user nor registry in session - if not current_registry and current_user.is_anonymous: - raise NotAuthorizedException(detail=messages.unauthorized_no_user_nor_registry) - # if there is a registry user in session - # check whether his token issued by the registry is valid - if current_registry and not current_user.is_anonymous: - if current_registry.name not in current_user.oauth_identity: - raise NotAuthorizedException( - detail=messages.unauthorized_user_without_registry_identity.format(current_registry.name), - authorization_url=url_for('oauth2provider.authorize', - name=current_registry.name, next=request.url)) + is_user_or_registry_authenticated() + return func(*args, **kwargs) + return wrapper + + +def authorized_by_session_or_apikey(func): + @wraps(func) + def wrapper(*args, **kwargs): + apiKey = request.headers.get('ApiKey', None) + if not apiKey: + authHeader = request.headers.get('Authorization', None) + if authHeader: + apiKey = authHeader.replace('ApiKey ', '') + if not apiKey: + raise NotAuthorizedException() + check_api_key(api_key=apiKey, required_scopes=()) + is_user_or_registry_authenticated() return func(*args, **kwargs) return wrapper diff --git a/lifemonitor/auth/templates/auth/github_settings.j2 b/lifemonitor/auth/templates/auth/github_settings.j2 index 9cecb47bc..43b09d341 100644 --- a/lifemonitor/auth/templates/auth/github_settings.j2 +++ b/lifemonitor/auth/templates/auth/github_settings.j2 @@ -1,4 +1,4 @@ -{% import 'auth/macros.j2' as macros %} +{% import 'macros.j2' as macros %}
diff --git a/lifemonitor/auth/templates/auth/identity_not_found.j2 b/lifemonitor/auth/templates/auth/identity_not_found.j2 new file mode 100644 index 000000000..121fbe04b --- /dev/null +++ b/lifemonitor/auth/templates/auth/identity_not_found.j2 @@ -0,0 +1,72 @@ +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} + +{% block body_class %} sidebar-mini sidebar-open login-page h-100 {% endblock %} + +{% block body %} + +
+ +{% endblock body %} \ No newline at end of file diff --git a/lifemonitor/auth/templates/auth/login.j2 b/lifemonitor/auth/templates/auth/login.j2 index 081868fc5..4bf2bd459 100644 --- a/lifemonitor/auth/templates/auth/login.j2 +++ b/lifemonitor/auth/templates/auth/login.j2 @@ -1,5 +1,5 @@ -{% extends 'auth/base.j2' %} -{% import 'auth/macros.j2' as macros %} +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} {% block body_class %} login-page {% endblock %} {% block body_style %} height: auto; {% endblock %} @@ -39,14 +39,27 @@ + +

Don’t have an account? @@ -56,4 +69,4 @@ -{% endblock body %} \ No newline at end of file +{% endblock body %} diff --git a/lifemonitor/auth/templates/auth/merge.j2 b/lifemonitor/auth/templates/auth/merge.j2 index 39a3992d7..5406e5e55 100644 --- a/lifemonitor/auth/templates/auth/merge.j2 +++ b/lifemonitor/auth/templates/auth/merge.j2 @@ -1,5 +1,5 @@ -{% extends 'auth/base.j2' %} -{% import 'auth/macros.j2' as macros %} +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} {% block body %}

Merge Account

If you want to merge another account into this one, log in to that account here. That account must have a password set.

diff --git a/lifemonitor/auth/templates/auth/notifications.j2 b/lifemonitor/auth/templates/auth/notifications.j2 index ed24a0c60..34a7f20dd 100644 --- a/lifemonitor/auth/templates/auth/notifications.j2 +++ b/lifemonitor/auth/templates/auth/notifications.j2 @@ -1,4 +1,4 @@ -{% import 'auth/macros.j2' as macros %} +{% import 'macros.j2' as macros %}
{{ notificationsForm.hidden_tag() }} diff --git a/lifemonitor/auth/templates/auth/profile.j2 b/lifemonitor/auth/templates/auth/profile.j2 index 98b4dbb83..34da6724a 100644 --- a/lifemonitor/auth/templates/auth/profile.j2 +++ b/lifemonitor/auth/templates/auth/profile.j2 @@ -1,5 +1,5 @@ -{% extends 'auth/base.j2' %} -{% import 'auth/macros.j2' as macros %} +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} {% block body_class %} login-page {% endblock %} {% block body_style %} {% if current_user.is_authenticated %} diff --git a/lifemonitor/auth/templates/auth/register.j2 b/lifemonitor/auth/templates/auth/register.j2 index e58adadc1..2656f794d 100644 --- a/lifemonitor/auth/templates/auth/register.j2 +++ b/lifemonitor/auth/templates/auth/register.j2 @@ -1,11 +1,11 @@ -{% extends 'auth/base.j2' %} -{% import 'auth/macros.j2' as macros %} +{% extends 'base.j2' %} +{% import 'macros.j2' as macros %} -{% block body_class %} sidebar-mini sidebar-open login-page {% endblock %} +{% block body_class %} sidebar-mini sidebar-open login-page h-100 {% endblock %} {% block body %} - diff --git a/lifemonitor/auth/templates/auth/registry_settings.j2 b/lifemonitor/auth/templates/auth/registry_settings.j2 index 7521ee2dc..9590f8e05 100644 --- a/lifemonitor/auth/templates/auth/registry_settings.j2 +++ b/lifemonitor/auth/templates/auth/registry_settings.j2 @@ -1,4 +1,4 @@ -{% import 'auth/macros.j2' as macros %} +{% import 'macros.j2' as macros %} %r", n, nrun, nrun.created_at) for run in runs: @@ -235,7 +240,7 @@ def test_get_runs_by_date(github_service: models.GithubTestingService, git_ref, runs[0].created_at.isoformat()))] logger.debug("Runs after: %r --- num: %r", runs_after, len(runs_after)) # check number of runs - assert len(runs_after) == (n + 1), "Unexpected number of runs" + assert len(runs_after) == (len(runs) - n), "Unexpected number of runs" # check run creation time for run in runs_after: logger.debug("Run: %r --> %r -- of array: %r", run, run.created_at, run in runs) @@ -308,16 +313,22 @@ def test_instance_builds_versioned_by_date( gh_workflow = github_service._get_gh_workflow(repository, workflow_id) logger.debug("Gh Workflow: %r", gh_workflow) - all_runs = list(run for run in github_service._list_workflow_runs(test_instance_one_version, limit=build_query_limit)) - logger.debug("Runs: %r", all_runs) + items_limit = 10 - instance_all_builds = github_service.get_test_builds(test_instance_one_version, limit=len(all_runs)) - logger.debug("Instance runs: %r", instance_all_builds) + # github_service._get_gh_workflow_runs( + all_runs = github_service._list_workflow_runs(test_instance_one_version, limit=items_limit) + for run in all_runs: + logger.debug("Run: {} created at {} updated at {} attempts {}".format(run, run.created_at, run.updated_at, run.raw_data.get("run_attempt"))) + assert len(all_runs) == 6, "Unexpected number of runs" + + instance_all_builds = github_service.get_test_builds(test_instance_one_version, limit=items_limit) + for instance in instance_all_builds: + logger.debug("Instance run: %r", instance) - assert len(instance_all_builds) == len(all_runs), "Unexpected number of runs for the instance revision" + assert len(instance_all_builds) == 10, "Unexpected number of runs for the instance revision" for run in all_runs: - logger.warning("Build %r created at %r updated at %r", run, run.created_at, run.updated_at) + logger.debug("Build %r created at %r updated at %r", run, run.created_at, run.updated_at) with cache.transaction(): branch_run_ids = [_.id for _ in all_runs] @@ -336,17 +347,20 @@ def test_instance_builds_versioned_by_date( # simulate latest version with at least one previous version with cache.transaction(): - builds_split = instance_all_builds[-1] - logger.error("Build split: %r", datetime.fromtimestamp(builds_split.timestamp)) + for b in instance_all_builds: + logger.debug("Instance: %r - %r", b, b.created_at) + logger.debug("Build split: %r", datetime.fromtimestamp(all_runs[2].created_at.timestamp())) v1 = MagicMock() - v1.created = datetime.fromtimestamp(builds_split.timestamp) + v1.created = datetime.fromtimestamp(all_runs[2].created_at.timestamp()) test_instance_one_version.test_suite.workflow_version.previous_version = v1 - test_instance_one_version.test_suite.workflow_version.created = v1.created + timedelta(minutes=3) + test_instance_one_version.test_suite.workflow_version.created = datetime.fromtimestamp(all_runs[1].created_at.timestamp()) + test_instance_one_version.test_suite.workflow_version.next_version = None assert test_instance_one_version.test_suite.workflow_version.previous_version - instance_builds = github_service.get_test_builds(test_instance_one_version, limit=len(all_runs)) - logger.debug("Instance runs: %r", instance_builds) + instance_builds = github_service.get_test_builds(test_instance_one_version, limit=items_limit) + logger.debug("Instance runs: %r --> count: %d", instance_builds, len(instance_builds)) + # raise RuntimeError("Runs: %r", instance_builds) instance_run_ids = [_.build_number for _ in instance_builds] found = [] @@ -356,23 +370,23 @@ def test_instance_builds_versioned_by_date( found.append(run) else: not_found.append(run) - logger.debug("Found: %r", [f"{x}: {x.updated_at}" for x in found]) - logger.debug("Not found: %r", [f"{x}: {x.updated_at}" for x in not_found]) + logger.debug("Found: %r", [f"{x}: {x.created_at}" for x in found]) + logger.debug("Not found: %r", [f"{x}: {x.created_at}" for x in not_found]) assert len(instance_builds) == (len(instance_all_builds) - 1), "Unexpected number of runs for the instance revision" # simulate an intermediate workflow version with cache.transaction(): - builds_split = instance_all_builds[-2] + # builds_split = all_runs[1] v2 = MagicMock() test_instance_one_version.test_suite.workflow_version.next_version = v2 - test_instance_one_version.test_suite.workflow_version.created = datetime.fromtimestamp(builds_split.timestamp) - v2.created = datetime.fromtimestamp(instance_all_builds[-3].timestamp) + test_instance_one_version.test_suite.workflow_version.created = datetime.fromtimestamp(all_runs[1].created_at.timestamp()) + v2.created = datetime.fromtimestamp(all_runs[0].created_at.timestamp()) - instance_builds = github_service.get_test_builds(test_instance_one_version, limit=len(all_runs)) + instance_builds = github_service.get_test_builds(test_instance_one_version, limit=items_limit) logger.debug("Instance runs: %r", instance_builds) - assert len(instance_builds) == 2, "Unexpected number of runs for the instance revision" + assert len(instance_builds) == 8, "Unexpected number of runs for the instance revision" @pytest.mark.skipif(not token, reason="Github token not set") diff --git a/tests/unit/cache/test_cache.py b/tests/unit/cache/test_cache.py index 8f6f5f377..44d42ecb7 100644 --- a/tests/unit/cache/test_cache.py +++ b/tests/unit/cache/test_cache.py @@ -24,12 +24,13 @@ from time import sleep from unittest.mock import MagicMock -import lifemonitor.api.models as models import pytest + +import lifemonitor.api.models as models from lifemonitor.cache import (IllegalStateException, Timeout, cache, init_cache, make_cache_key) from tests import utils -from tests.unit.test_utils import SerializableMock +from tests.utils import SerializableMock logger = logging.getLogger(__name__) diff --git a/tests/unit/schemas/data.yaml b/tests/unit/schemas/data.yaml new file mode 100644 index 000000000..815329e64 --- /dev/null +++ b/tests/unit/schemas/data.yaml @@ -0,0 +1,50 @@ +# worfklow name (override name defined on the RO-Crate metadata) +# name: MyWorkflow +# worfklow visibility +public: False + +# Issue Checker Settings +issues: + # Enable/Disable issue checker + # The list of issue types can be found @ /workflows/issues + # (e.g., https://api.lifemonitor.eu/workflows/issues) + check: true + # csv of issues to check (all included by default) + # include: [Template.MyIssue, lm.MissingLMConfigFile, repo_layout.RepositoryNotInitialised, metadata.MissingWorkflowName, repo_layout.MissingWorkflowFile, repo_layout.MissingROCrateFile, experimental.OutdatedROCrateFile, repo_layout.MissingROCrateWorkflowFile] + # csv of issues to ignore (none ignored by default) + # exclude: [Template.MyIssue, lm.MissingLMConfigFile, repo_layout.RepositoryNotInitialised, metadata.MissingWorkflowName, repo_layout.MissingWorkflowFile, repo_layout.MissingROCrateFile, experimental.OutdatedROCrateFile, repo_layout.MissingROCrateWorkflowFile] +# Github Integration Settings +push: + branches: + # Define the list of branches to watch + # - name: feature/XXX # wildcards can be used to specify branches (e.g., feature/*) + # update_registries: ["wfhubdev"] # available registries are listed + # # by the endpoint `/registries` + # # (e.g., https://api.lifemontor.eu/registries) + # lifemonitor_instance: development # uncomment to use the 'development' instance of LifeMonitor + # # (the 'production' instance is used by default) + - name: "main" + update_registries: [] + enable_notifications: true + # lifemonitor_instance: development + # - name: "develop" + # update_registries: [] + # enable_notifications: true + # lifemonitor_instance: development + + tags: + # Define the list of tags to watch + # - name: v*.*.* # wildcards can be used to specify tags (e.g., feature/*) + # update_registries: ["wfhub"] # available registries are listed + # # by the endpoint `/registries` + # # (e.g., https://api.lifemontor.eu/registries) + # lifemonitor_instance: development # uncomment to use the 'development' instance of LifeMonitor + # # (the 'production' instance is used by default) + - name: "v*.*.*" + update_registries: [wfhubdev, wfhubprod, seek] + enable_notifications: true + lifemonitor_instance: developmentx + - name: "ok" + update_registries: [wfhubdev, wfhubprod, seek] + enable_notifications: true + # lifemonitor_instance: development diff --git a/tests/unit/schemas/test_lm_schema_validator.py b/tests/unit/schemas/test_lm_schema_validator.py new file mode 100644 index 000000000..214f33155 --- /dev/null +++ b/tests/unit/schemas/test_lm_schema_validator.py @@ -0,0 +1,103 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import json +import logging +import os +from typing import Dict + +import pytest +import yaml + +from lifemonitor.schemas.validators import ConfigFileValidator, ValidationError, ValidationResult + +__current_path__ = os.path.dirname(os.path.realpath(__file__)) + + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def data() -> Dict: + with open(f'{__current_path__}/data.yaml', 'r') as f: + return yaml.unsafe_load(f) + + +@pytest.fixture +def schema() -> Dict: + with open(f'{__current_path__}/../../../lifemonitor/schemas/lifemonitor.json', 'r') as f: + return json.load(f) + + +def test_schema_loader(schema): + assert sorted(ConfigFileValidator.schema.items()) == sorted(schema.items()), "Invalid schema" + + +def test_ref_solve(schema): + + ref = schema['definitions']['push_ref'] + logger.debug(ref) + push_ref = ConfigFileValidator.find_definition(schema, "#/definitions/push_ref") + assert push_ref == ref, "Invalid ref object" + assert push_ref['properties']['update_registries']['default'] == [], "Invalid default value" + + +def test_valid_schema(data, schema): + result: ValidationResult = ConfigFileValidator.validate(data) + assert result is not None, "Result should be empty" + assert isinstance(result, ValidationResult), "Unexpected validation response: 'valid' property not found" + assert type(result.valid == bool), "'valid' property should be a boolean value" + assert result.valid is True, "Unexpected validation response" + + +def test_default_for_public_property(data, schema): + # remove public property from data + del data['public'] + assert 'public' not in data, "public property should not be set on data" + result: ValidationResult = ConfigFileValidator.validate(data) + assert isinstance(result, ValidationResult), "Unexpected validation response: 'valid' property not found" + assert result.input_data == data, "Unexpected input data" + assert 'public' not in result.input_data, "Public property should not be set on data" + print("Output data: %r" % result.output_data) + assert result.output_data['public'] == schema['properties']['public']['default'], "Public property should be initialized with the defaul value" + + +def test_default_update_registries_of_branch(data, schema): + + del data['push']['branches'][0]['update_registries'] + assert 'update_registries' not in data['push']['branches'][0], "update_registries of branch main should not be set" + result: ValidationError = ConfigFileValidator.validate(data) + assert result is not None, "Result should be empty" + assert result.valid is True, "Data should be valid" + print(json.dumps(result.output_data, indent=2)) + assert 'update_registries' in result.output_data['push']['branches'][0],\ + "update_registries should be automatically initialized" + assert result.output_data['push']['branches'][0]['update_registries'] == [], \ + "update_registries should be automatically initialized with default values" + + +def test_missing_branch_name(data): + del data['push']['branches'][0]['name'] + result: ValidationError = ConfigFileValidator.validate(data) + assert result is not None, "Result should be empty" + assert isinstance(result, ValidationError), "Unexpected validation response: 'valid' property not found" + assert type(result.valid == bool), "'valid' property should be a boolean value" + assert result.valid is False, "Validation should fail" + assert result.message == "'name' is a required property" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index afbe33953..68c20eca0 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -20,20 +20,118 @@ import os import tempfile -from unittest.mock import MagicMock, Mock + +import pytest import lifemonitor.exceptions as lm_exceptions import lifemonitor.utils as utils -import pytest def test_download_url_404(): with tempfile.TemporaryDirectory() as d: - with pytest.raises(lm_exceptions.DownloadException) as excinfo: + with pytest.raises(lm_exceptions.DownloadException) as exec_info: _ = utils.download_url('http://httpbin.org/status/404', os.path.join(d, 'get_404')) - assert excinfo.value.status == 404 + assert exec_info.value.status == 404 + + +def test_datetime_to_isoformat(): + """Test the datetime_to_isoformat function.""" + from datetime import datetime + + # test with a datetime with microseconds + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + assert utils.datetime_to_isoformat(dt) == "2020-01-01T00:00:00.123456Z" + + # test with a datetime without microseconds + dt = datetime(2020, 1, 1, 0, 0, 0) + assert utils.datetime_to_isoformat(dt) == "2020-01-01T00:00:00Z" + + +def test_isoformat_to_datetime(): + """Test the isoformat_to_datetime function.""" + from datetime import datetime + + # test with a datetime with microseconds + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds + dt = datetime(2020, 1, 1, 0, 0, 0) + iso = "2020-01-01T00:00:00Z" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds and without Z + dt = datetime(2020, 1, 1, 0, 0, 0) + iso = "2020-01-01T00:00:00" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds and without Z and without seconds + dt = datetime(2020, 1, 1, 0, 0) + iso = "2020-01-01T00:00" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds and without Z and without seconds and without minutes + dt = datetime(2020, 1, 1, 0) + iso = "2020-01-01T00" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds and without Z and without seconds and without minutes and without hours + dt = datetime(2020, 1, 1) + iso = "2020-01-01" + assert utils.isoformat_to_datetime(iso) == dt + + # test with a datetime without microseconds and without Z and without seconds and without minutes and without hours and without day + dt = datetime(2020, 1, 1) + iso = "2020-01" + pytest.raises(ValueError, utils.isoformat_to_datetime, iso) + + # test with a datetime without microseconds and without Z and without seconds and without minutes and without hours and without day and without month + dt = datetime(2020, 1, 1) + iso = "2020" + pytest.raises(ValueError, utils.isoformat_to_datetime, iso) + + +def test_parse_date_interval(): + """Test the parse_date_interval function.""" + from datetime import datetime + + # test with a datetime with microseconds and operator <= + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + operator, start_date, end_date = utils.parse_date_interval(f"<={iso}") + assert operator == "<=" + assert start_date is None + assert end_date == dt + + # test with a datetime with microseconds and operator >= + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + operator, start_date, end_date = utils.parse_date_interval(f">={iso}") + assert operator == ">=" + assert start_date == dt + assert end_date is None + + # test with a datetime with microseconds and operator < + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + operator, start_date, end_date = utils.parse_date_interval(f"<{iso}") + assert operator == "<" + assert start_date is None + assert end_date == dt + # test with a datetime with microseconds and operator > + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + operator, start_date, end_date = utils.parse_date_interval(f">{iso}") + assert operator == ">" + assert start_date == dt + assert end_date is None -class SerializableMock(MagicMock): - def __reduce__(self): - return (Mock, ()) + # test with a datetime with microseconds and operator .. + dt = datetime(2020, 1, 1, 0, 0, 0, 123456) + iso = "2020-01-01T00:00:00.123456Z" + operator, start_date, end_date = utils.parse_date_interval(f"{iso}..{iso}") + assert operator == ".." + assert start_date == dt + assert end_date == dt diff --git a/tests/utils.py b/tests/utils.py index ba696a051..4b58a70d3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,6 +19,7 @@ # SOFTWARE. import logging +from unittest.mock import MagicMock, Mock import lifemonitor.db as lm_db from lifemonitor.api import models @@ -151,3 +152,8 @@ def not_shared_workflows(user1, user2, skip=None): def get_workflow_data(wf_uuid): return LifeMonitor.get_instance().get_workflow(wf_uuid) + + +class SerializableMock(MagicMock): + def __reduce__(self): + return (Mock, ()) diff --git a/run.py b/ws.py similarity index 73% rename from run.py rename to ws.py index 22d1c00d7..95e90d018 100644 --- a/run.py +++ b/ws.py @@ -18,8 +18,17 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from lifemonitor import app +import logging -if __name__ == '__main__': - """ Start development server""" - app.create_app().run(host="0.0.0.0", port=8000) +from lifemonitor.app import create_app +from lifemonitor.ws import initialise_ws +from lifemonitor.ws.io import start_brodcaster + +# initialise logger +logger = logging.getLogger(__name__) + + +# create an app instance +application = create_app(init_app=True, load_jobs=False) +socketIO = initialise_ws(application) +start_brodcaster(application, max_age=5) # use default ws_channel