diff --git a/Makefile b/Makefile index 0b3bb613..64bb9792 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,8 @@ permissions: certs prometheus.dev.yml \ settings.conf \ tests/config/registries/seek/nginx.conf \ - tests/config/registries/seek/doorkeeper.rb + tests/config/registries/seek/doorkeeper.rb \ + && mkdir -p /tmp/lifemonitor-logs aux_images: tests/config/registries/seek/seek.Dockerfile certs @@ -223,7 +224,8 @@ start-dev: images compose-files dev reset_compose permissions ## Start LifeMonit start-testing: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment @printf "\n$(bold)Starting testing services...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ - echo "$$($(docker_compose) $${base} \ + echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ + $(docker_compose) $${base} \ -f docker-compose.extra.yml \ -f docker-compose.base.yml \ -f docker-compose.monitoring.yml \ diff --git a/docker-compose.base.yml b/docker-compose.base.yml index ee0d60d0..5e682008 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -1,5 +1,3 @@ -version: "3.5" - services: db: # Docker image docs: https://github.com/bitnami/bitnami-docker-postgresql @@ -65,6 +63,7 @@ services: - "./instance:/lm/instance:ro" - "./settings.conf:/lm/settings.conf:ro" # default settings - "data_workflows:/var/data/lm" + - "/tmp/lifemonitor-logs:/var/log/lm" networks: - life_monitor diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index df1310b3..1c880321 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: "3.5" - services: db: environment: diff --git a/docker-compose.extra.yml b/docker-compose.extra.yml index 4cd89b6c..2a6c5530 100644 --- a/docker-compose.extra.yml +++ b/docker-compose.extra.yml @@ -1,5 +1,3 @@ -version: "3.5" - services: seek: image: crs4/lifemonitor-tests:seek diff --git a/docker-compose.maintenance.yml b/docker-compose.maintenance.yml new file mode 100644 index 00000000..17646c29 --- /dev/null +++ b/docker-compose.maintenance.yml @@ -0,0 +1,61 @@ +services: + db: + environment: + - "ALLOW_EMPTY_PASSWORD=yes" + + console: + image: crs4/lifemonitor + restart: "unless-stopped" + depends_on: + - "db" + - "init" + - "redis" + entrypoint: /bin/bash + command: | + -c "sleep infinity" + user: "${USER_UID}:${USER_GID}" + env_file: + - settings.conf + environment: + - "FLASK_ENV=development" + - "TEMPLATES_AUTO_RELOAD=True" + - "HOME=/lm" + # Normally, OAuthLib will raise an InsecureTransportError if you attempt to use OAuth2 over HTTP, + # rather than HTTPS. Setting this environment variable will prevent this error from being raised. + # This is mostly useful for local testing, or automated tests. Never set this variable in production. + - "AUTHLIB_INSECURE_TRANSPORT=1" + # - "SERVER_NAME=lm:8000" + volumes: + - "./certs:/certs:ro" + - "./instance:/lm/instance:ro" + - "./settings.conf:/lm/settings.conf:ro" # default settings + - "./:/lm" + - "data_workflows:/var/data/lm" + ports: + - "8000" + networks: + - life_monitor + + init: + user: "${USER_UID}:${USER_GID}" + environment: + - "FLASK_ENV=development" + # Normally, OAuthLib will raise an InsecureTransportError if you attempt to use OAuth2 over HTTP, + # rather than HTTPS. Setting this environment variable will prevent this error from being raised. + # This is mostly useful for local testing, or automated tests. Never set this variable in production. + - "AUTHLIB_INSECURE_TRANSPORT=1" + - "HOME=/lm" + volumes: + - "./:/lm" + + lm: + environment: + - "FLASK_ENV=maintenance" + + ws_server: + environment: + - "FLASK_ENV=maintenance" + + worker: + environment: + - "FLASK_ENV=maintenance" diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 2e17806e..56983999 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -1,5 +1,3 @@ -version: "3.5" - services: prometheus: image: prom/prometheus:v2.48.0 diff --git a/docker-compose.test.yml b/docker-compose.test.yml index c2fe73b1..2389352b 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,5 +1,3 @@ -version: "3.5" - services: lmtests: image: crs4/lifemonitor @@ -34,6 +32,7 @@ services: - "./certs:/certs:ro" - "./instance:/lm/instance:ro" - "./settings.conf:/lm/settings.conf:ro" # default settings + - "/tmp/lifemonitor-logs:/var/log/lm" ports: - "8000" networks: diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index 55f9b0ac..6b2b637f 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster as base +FROM python:3.10-slim-buster AS base # Install base requirements RUN apt-get update -q \ @@ -100,7 +100,7 @@ RUN find /lm/lifemonitor/ -type d -exec chmod a+r {} \; ################################################################## ## Node Stage ################################################################## -FROM node:14.16.0-alpine3.12 as node +FROM node:14.16.0-alpine3.12 AS node RUN mkdir -p /static && apk add --no-cache bash python3 make g++ \ @@ -119,7 +119,7 @@ RUN npm run production ################################################################## ## Target Stage ################################################################## -FROM base as target +FROM base AS target # Set software and build number ARG SW_VERSION diff --git a/docker/worker_entrypoint.sh b/docker/worker_entrypoint.sh index d82d65a4..5e0305db 100755 --- a/docker/worker_entrypoint.sh +++ b/docker/worker_entrypoint.sh @@ -41,9 +41,10 @@ export dramatiq_prom_db="${PROMETHEUS_MULTIPROC_DIR}" log "Starting task queue worker container" debug_log "PROMETHEUS_MULTIPROC_DIR = ${PROMETHEUS_MULTIPROC_DIR}" -if [[ -n "${DEBUG:-}" ]]; then +verbose="" +if [[ -n "${VERBOSE:-}" ]]; then verbose='--verbose' - log "Debug Mode Enabled" + echo "Verbose Mode Enabled" fi if [[ ${FLASK_ENV} == "development" ]]; then @@ -94,4 +95,4 @@ done log "Worker exiting with exit code ${exit_code}" -exit ${exit_code} \ No newline at end of file +exit ${exit_code} diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 1ae3d21f..c4611c5c 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -12,7 +12,7 @@ version: 0.12.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.14.0 +appVersion: 0.15.0 # Chart dependencies dependencies: diff --git a/k8s/templates/backup.job.yaml b/k8s/templates/backup.job.yaml index f320994e..52b5facb 100644 --- a/k8s/templates/backup.job.yaml +++ b/k8s/templates/backup.job.yaml @@ -1,5 +1,5 @@ {{- if .Values.backup.enabled -}} -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: {{ include "chart.fullname" . }}-backup @@ -58,4 +58,4 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} backoffLimit: 4 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/k8s/templates/data-backup.pvc.yaml b/k8s/templates/data-backup.pvc.yaml index 2ab35b82..3e93460b 100644 --- a/k8s/templates/data-backup.pvc.yaml +++ b/k8s/templates/data-backup.pvc.yaml @@ -5,6 +5,8 @@ kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.backup" . }} namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": keep spec: storageClassName: {{ .Values.global.readWriteManyStorageClass }} accessModes: diff --git a/k8s/templates/data-workflows.pvc.yaml b/k8s/templates/data-workflows.pvc.yaml index d2092fa0..253bbeb5 100644 --- a/k8s/templates/data-workflows.pvc.yaml +++ b/k8s/templates/data-workflows.pvc.yaml @@ -5,6 +5,8 @@ kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.workflows" . }} namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep spec: storageClassName: {{ .Values.global.readWriteManyStorageClass }} accessModes: diff --git a/k8s/templates/init.job.yaml b/k8s/templates/init.job.yaml index 83b50aff..b27b63b9 100644 --- a/k8s/templates/init.job.yaml +++ b/k8s/templates/init.job.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.maintenanceMode.enabled -}} apiVersion: batch/v1 kind: Job metadata: @@ -47,3 +48,4 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} backoffLimit: 4 +{{- end }} diff --git a/k8s/values.yaml b/k8s/values.yaml index 35636f22..4c3151c2 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -288,7 +288,7 @@ worker: - name: github # image: *lifemonitorImage # - name: heartbeat - # image: *lifemonitorImage + # image: *lifemonitorImage - name: notifications # image: *lifemonitorImage - name: builds @@ -296,7 +296,7 @@ worker: - name: workflows # image: *lifemonitorImage # - name: metrics - # image: *lifemonitorImage + # image: *lifemonitorImage - name: ws # image: *lifemonitorImage diff --git a/lifemonitor/api/controllers.py b/lifemonitor/api/controllers.py index 16d68176..1b8c7ef5 100644 --- a/lifemonitor/api/controllers.py +++ b/lifemonitor/api/controllers.py @@ -249,7 +249,9 @@ def registry_user_workflows_get(user_id, status=False, versions=False): workflows = lm.get_user_registry_workflows(identity.user, current_registry) logger.debug("registry_user_workflows_get. Got %s workflows (user: %s)", len(workflows), current_user) return serializers.ListOfWorkflows(workflow_status=status, workflow_versions=versions).dump(workflows) - except OAuthIdentityNotFoundException: + except OAuthIdentityNotFoundException as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) return lm_exceptions.report_problem(401, "Unauthorized", detail=messages.no_user_oauth_identity_on_registry .format(user_id, current_registry.name)) diff --git a/lifemonitor/api/models/repositories/config.py b/lifemonitor/api/models/repositories/config.py index 51fcf375..86f75dea 100644 --- a/lifemonitor/api/models/repositories/config.py +++ b/lifemonitor/api/models/repositories/config.py @@ -150,7 +150,11 @@ def registries(self) -> List[models.WorkflowRegistry]: for rfs in self._get_refs_list(): for r in rfs.get("update_registries", []): if not registries.get(r, None): - registries[r] = models.WorkflowRegistry.find_by_client_name(r) + r_model = models.WorkflowRegistry.find_by_client_name(r) + if r_model is None: + logger.warning("Unable to find registry: %r", r) + else: + registries[r] = r_model return list(registries.values()) def get_ref_registries(self, ref_type: str, tag: str) -> List[models.WorkflowRegistry]: diff --git a/lifemonitor/api/services.py b/lifemonitor/api/services.py index 3c96094b..58bf3ee6 100644 --- a/lifemonitor/api/services.py +++ b/lifemonitor/api/services.py @@ -626,7 +626,8 @@ def find_registry_user_identity(registry: models.WorkflowRegistry, raise ValueError("external_id and internal_id cannot be both None") if internal_id: return OAuthIdentity.find_by_user_id(internal_id, registry.name) - return OAuthIdentity.find_by_provider_user_id(external_id, registry.name) + assert registry, "The registry should be defined" + return OAuthIdentity.find_by_provider_user_id(external_id, registry.client_name) @staticmethod def add_workflow_registry(type, name, diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index 0eed0ede..cba44ce8 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.14.0", + "version": "0.15.0", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/requirements.txt b/requirements.txt index f053f76c..8c76d6ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,61 +1,59 @@ -Authlib~=0.15.4 -apscheduler==3.10.1 -connexion[swagger-ui]~=2.14.2 -dramatiq[redis,watch]==1.14.2 -email-validator~=1.1.3 -boto3~=1.24.80 +Authlib~=0.15.6 +apscheduler==3.10.4 +boto3~=1.35.9 Bcrypt-Flask==1.0.2 -giturlparse~=0.10.0 click-option-group~=0.5.5 +connexion[swagger-ui]~=2.14.2 cryptography>=41.0.3 dnspython==2.6.1 -flask-cors==3.0.10 -flask-marshmallow~=0.14.0 -flask-restful==0.3.9 +dramatiq[redis,watch]==1.14.2 +email-validator~=2.2.0 +eventlet==0.36.1 +flask-cors==4.0.1 flask-login~=0.6.2 +flask-marshmallow~=0.14.0 +flask-restful==0.3.10 flask-shell-ipython==0.5.1 -flask-wtf~=1.1.1 -Flask-APScheduler==1.12.4 +flask-wtf~=1.2.0 +Flask-APScheduler==1.13.1 +Flask-Mail~=0.10.0 +Flask-Migrate==4.0.7 Flask-SQLAlchemy==2.5.1 -Flask-Migrate==3.1.0 -Flask-Mail~=0.9.1 Flask~=2.2.5 -gunicorn~=20.1.0 -itsdangerous~=2.1.0 +flask-socketio==5.3.6 +gevent-websocket==0.10.1 +giturlparse~=0.12.0 +gunicorn~=23.0.0 +itsdangerous~=2.2.0 jwt==1.3.1 -jsonschema==3.2.0 +jsonschema==4.23.0 loginpass==0.5 marshmallow-sqlalchemy~=0.28.2 +networkx~=3.3 nf-core~=2.5.0 +pika~=1.3.2 prometheus-flask-exporter~=0.22 -pygit2~=1.12.1 psycopg2-binary~=2.9.6 -pyopenssl==23.2.0 -pytest-mock~=3.11.1 +pygit2~=1.15.1 +pyopenssl==24.2.1 pytest~=7.4.0 +pytest-mock~=3.11.1 pytest-rerunfailures~=12.0.0 pytest-xdist~=3.3.1 -python-dotenv~=0.19.0 +python-dotenv~=1.0.1 python-jenkins==1.7.0 python-redis-lock~=4.0.0 PyGithub==1.55 PyYAML~=6.0.1 -pika~=1.2.0 redis~=4.5.5 +repo2rocrate~=0.1.2 requests~=2.31.0 +rich~=13.8.0 +rocrate==0.10.0 snakemake==6.13.1 -networkx~=3.1.0 -rich~=13.4.1 -rocrate==0.8.0 SQLAlchemy==1.4.41 -wheel~=0.40.0 +urllib3==1.26.19 +uwsgi==2.0.26 +watchdog==4.0.2 +wheel~=0.44.0 Werkzeug~=2.2.3 -repo2rocrate~=0.1.2 -urllib3==1.26.18 -flask-socketio==5.3.4 -gevent-websocket==0.10.1 -eventlet==0.35.2 -uwsgi==2.0.22 -# watchdog version 2.3 introduces a change that breaks Flask in development mode. -# Don't use watchdog > 2.2.1 until up update Flask to something >= 2.2 -watchdog==2.2.1 diff --git a/specs/api.yaml b/specs/api.yaml index df3854f9..182630f3 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.14.0" + version: "0.15.0" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.14.0 of API. + Version 0.15.0 of API. tags: - name: GitHub Integration