release v0.5.0

.dockerignore

@@ -8,6 +8,7 @@
 **/__pycache__
 **/.npm
 *.pyc
+data
 docker/Dockerfile
 package-lock.json
 **/node_modules

.gitignore

@@ -18,6 +18,7 @@ instance
 *.pyc
 ./certs
 **/node_modules
+data
 lifemonitor/static/dist
 docker-compose.yml
 utils/certs/data

docker-compose.base.yml

@@ -40,6 +40,7 @@ services:
       - "./certs:/certs:ro"
       - "./instance:/lm/instance:ro"
       - "./settings.conf:/lm/settings.conf:ro" # default settings
+      - "data_workflows:/var/data/lm"
     networks:
       - life_monitor
 
@@ -63,6 +64,7 @@ services:
       - "./certs:/certs:ro"
       - "./instance:/lm/instance:ro"
       - "./settings.conf:/lm/settings.conf:ro" # default settings
+      - "data_workflows:/var/data/lm"
       - type: volume
         source: data_static_files
         target: /lm/lifemonitor/static
@@ -85,6 +87,7 @@ services:
       - "./certs:/certs:ro"
       - "./instance:/lm/instance:ro"
       - "./settings.conf:/lm/settings.conf:ro" # default settings
+      - "data_workflows:/var/data/lm"
     networks:
       - life_monitor
 
@@ -106,6 +109,7 @@ volumes:
   data_static_files:
   data_specs:
   data_redis:
+  data_workflows:
 
 networks:
   life_monitor:

docker/lifemonitor.Dockerfile

@@ -46,6 +46,12 @@ RUN chmod 755 \
 # Set the container entrypoint
 ENTRYPOINT /usr/local/bin/lm_entrypoint.sh
 
+# Prepare data folder
+RUN mkdir -p /var/data/lm \
+    && chown -R lm:lm /var/data/lm \
+    && ln -s /var/data/lm /lm/data \
+    && chown -R lm:lm /lm/data
+
 # Set the default user
 USER lm
 

k8s/Chart.lock

@@ -5,5 +5,8 @@ dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 10.1.1
-digest: sha256:386060b432509d7925e2d620559d44d0efbe869184049da1fc12cfd86b0b5550
-generated: "2021-04-25T15:26:03.079993489Z"
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.3.2
+digest: sha256:e69b28d1eb2d1b5e7fef36eb02b99cf305f52af14e4d2bda033eaf7544ef498b
+generated: "2021-10-21T16:46:24.046062+02:00"

k8s/Chart.yaml

@@ -7,12 +7,12 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.0
+version: 0.5.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.4.3
+appVersion: 0.5.0
 
 # Chart dependencies
 dependencies:

k8s/pvc-backend-data.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-api-workflows
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi

k8s/templates/_helpers.tpl

@@ -110,6 +110,9 @@ Define volumes shared by some pods.
 - name: lifemonitor-settings
   secret:
     secretName: {{ include "chart.fullname" . }}-settings
+- name: lifemonitor-data
+  persistentVolumeClaim:
+    claimName: data-{{- .Release.Name -}}-workflows
 {{- end -}}
 
 {{/*
@@ -122,4 +125,6 @@ Define mount points shared by some pods.
 - name: lifemonitor-settings
   mountPath: "/lm/settings.conf"
   subPath: settings.conf
+- name: lifemonitor-data
+  mountPath: "/var/data/lm"
 {{- end -}}

k8s/templates/job-init.yaml

@@ -3,7 +3,9 @@ kind: Job
 metadata:
   name: {{ include "chart.fullname" . }}-init
   labels:
-    {{- include "chart.labels" . | nindent 4 }}
+    app.kubernetes.io/name: {{ include "chart.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
   template:
     spec:

k8s/values.yaml

@@ -107,6 +107,7 @@ lifemonitor:
 
   persistence:
     storageClass: *storageClass
+    storageSize: 8Gi
 
   # Enable/Disable the pod to test connection to the LifeMonitor back-end
   enableTestConnection: false

lifemonitor/api/controllers.py

@@ -18,6 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import os
+import base64
 import logging
 import tempfile
 
@@ -62,6 +64,35 @@ def workflow_registries_get_by_uuid(registry_uuid):
     return serializers.WorkflowRegistrySchema().dump(registry)
 
 
+@authorized
+@cached(timeout=Timeout.REQUEST)
+def registry_index(registry_uuid):
+    if not current_user:
+        return lm_exceptions.report_problem(401, "Unauthorized")
+    registry = lm.get_workflow_registry_by_uuid(registry_uuid)
+    if not registry:
+        return lm_exceptions.report_problem(404, "Not Found",
+                                            detail=messages.no_registry_found.format(registry_uuid))
+    workflows = registry.get_index(current_user)
+    return serializers.ListOfRegistryIndexItemsSchema().dump(workflows)
+
+
+@authorized
+@cached(timeout=Timeout.REQUEST)
+def registry_index_workflow(registry_uuid, registry_workflow_identifier):
+    if not current_user:
+        return lm_exceptions.report_problem(401, "Unauthorized")
+    registry = lm.get_workflow_registry_by_uuid(registry_uuid)
+    if not registry:
+        return lm_exceptions.report_problem(404, "Not Found",
+                                            detail=messages.no_registry_found.format(registry_uuid))
+    workflow = registry.get_index_workflow(current_user, registry_workflow_identifier)
+    if not workflow:
+        return lm_exceptions.report_problem(404, "Not Found",
+                                            detail=messages.workflow_not_found.format(registry_workflow_identifier, 'latest'))
+    return serializers.RegistryIndexItemSchema().dump(workflow)
+
+
 @authorized
 @cached(timeout=Timeout.REQUEST)
 def workflow_registries_get_current():
@@ -306,10 +337,23 @@ def workflows_post(body, _registry=None, _submitter_id=None):
                                                 detail=messages.input_data_missing)
 
     roc_link = body.get('roc_link', None)
-    if not registry and not roc_link:
-        return lm_exceptions.report_problem(400, "Bad Request", extra_info={"missing input": "roc_link"},
+    encoded_rocrate = body.get('rocrate', None)
+    if not registry and not roc_link and not encoded_rocrate:
+        return lm_exceptions.report_problem(400, "Bad Request", extra_info={"missing input": "roc_link OR rocrate"},
                                             detail=messages.input_data_missing)
 
+    temp_rocrate_file = None
+    if encoded_rocrate:
+        try:
+            rocrate = base64.b64decode(encoded_rocrate)
+            temp_rocrate_file = tempfile.NamedTemporaryFile(delete=False, prefix="/tmp/")
+            temp_rocrate_file.write(rocrate)
+            roc_link = f"tmp://{temp_rocrate_file.name}"
+            logger.debug("ROCrate written to %r", temp_rocrate_file.name)
+        except Exception as e:
+            return lm_exceptions.report_problem(400, "Bad Request", extra_info={"exception": str(e)},
+                                                detail=messages.decode_ro_crate_error)
+
     submitter = current_user if current_user and not current_user.is_anonymous else None
     if not submitter:
         try:
@@ -366,6 +410,12 @@ def workflows_post(body, _registry=None, _submitter_id=None):
     except Exception as e:
         logger.exception(e)
         raise lm_exceptions.LifeMonitorException(title="Internal Error", detail=str(e))
+    finally:
+        if roc_link and roc_link.startswith("tmp://"):
+            try:
+                os.remove(roc_link.replace('tmp://', ''))
+            except Exception as e:
+                logger.error("Error deleting temp rocrate: %r", str(e))
 
 
 @authorized

lifemonitor/api/models/__init__.py

@@ -30,7 +30,7 @@
 from .status import Status, AggregateTestStatus, WorkflowStatus, SuiteStatus
 
 # 'registries' package
-from .registries import WorkflowRegistry, WorkflowRegistryClient
+from .registries import RegistryWorkflow, WorkflowRegistry, WorkflowRegistryClient
 
 # 'workflows' package
 from .workflows import Workflow, WorkflowVersion
@@ -72,6 +72,7 @@
     "WorkflowRegistryClient",
     "WorkflowStatus",
     "WorkflowVersion",
+    "RegistryWorkflow"
 ]
 
 # set module level logger

lifemonitor/api/models/registries/__init__.py

@@ -21,9 +21,9 @@
 from __future__ import annotations
 
 from lifemonitor.utils import ClassManager
-from .registry import WorkflowRegistry, WorkflowRegistryClient
+from .registry import RegistryWorkflow, WorkflowRegistry, WorkflowRegistryClient
 
 
-__all__ = [WorkflowRegistry, WorkflowRegistryClient] + \
+__all__ = [RegistryWorkflow, WorkflowRegistry, WorkflowRegistryClient] + \
     ClassManager('lifemonitor.api.models.registries',
                  class_suffix="WorkflowRegistry", skip=["registry"], lazy=False).get_classes()

lifemonitor/api/models/registries/registry.py

@@ -41,6 +41,51 @@
 logger = logging.getLogger(__name__)
 
 
+class RegistryWorkflow(object):
+    _registry: WorkflowRegistry
+    _identifier: str
+    _name: str
+    _latest_version: str
+    _versions: List[str] = None
+
+    def __init__(self,
+                 registry: WorkflowRegistry,
+                 identifier: str,
+                 name: str,
+                 latest_version: str = None,
+                 versions: List[str] = None) -> None:
+        self._registry = registry
+        self._identifier = identifier
+        self._name = name
+        self._latest_version = latest_version
+        if versions:
+            self._versions = versions.copy()
+
+    @property
+    def registry(self) -> WorkflowRegistry:
+        return self._registry
+
+    @property
+    def identifier(self) -> str:
+        return self._identifier
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def latest_version(self) -> str:
+        return self._latest_version
+
+    @property
+    def versions(self) -> List[str]:
+        return self._versions.copy()
+
+    @property
+    def external_link(self) -> str:
+        return self._registry.get_external_link(self._identifier, self._latest_version)
+
+
 class WorkflowRegistryClient(ABC):
 
     client_types = ClassManager('lifemonitor.api.models.registries', class_suffix="WorkflowRegistryClient", skip=["registry"])
@@ -78,6 +123,12 @@ def _get(self, user, *args, **kwargs):
         response.raise_for_status()
         return response
 
+    def get_index(self, user: auth_models.User) -> List[RegistryWorkflow]:
+        pass
+
+    def get_index_workflow(self, user: auth_models.User, workflow_identifier: str) -> RegistryWorkflow:
+        pass
+
     def download_url(self, url, user, target_path=None):
         return download_url(url, target_path,
                             authorization=f'Bearer {self._get_access_token(user.id)["access_token"]}')
@@ -240,6 +291,12 @@ def get_user_workflows(self, user: auth_models.User) -> List[models.Workflow]:
     def get_user_workflow_versions(self, user: auth_models.User) -> List[models.WorkflowVersion]:
         return self.client.filter_by_user(self.registered_workflow_versions, user)
 
+    def get_index(self, user: auth_models.User) -> List[RegistryWorkflow]:
+        return self.client.get_index(user)
+
+    def get_index_workflow(self, user: auth_models.User, workflow_identifier: str) -> RegistryWorkflow:
+        return self.client.get_index_workflow(user, workflow_identifier)
+
     @classmethod
     def all(cls) -> List[WorkflowRegistry]:
         return cls.query.all()

lifemonitor/api/models/registries/seek.py

@@ -21,13 +21,16 @@
 from __future__ import annotations
 
 import logging
-from typing import Union
+from typing import List, Union
 
+import requests
 from lifemonitor.api import models
 from lifemonitor.auth.models import User
-from lifemonitor.exceptions import EntityNotFoundException
+from lifemonitor.exceptions import (EntityNotFoundException,
+                                    LifeMonitorException)
 
-from .registry import WorkflowRegistry, WorkflowRegistryClient
+from .registry import (RegistryWorkflow, WorkflowRegistry,
+                       WorkflowRegistryClient)
 
 # set module level logger
 logger = logging.getLogger(__name__)
@@ -63,8 +66,26 @@ def get_workflow_metadata(self, user, w: Union[models.WorkflowVersion, str]):
             raise RuntimeError(f"ERROR: unable to get workflow (status code: {r.status_code})")
         return r.json()['data']
 
+    def get_index(self, user: User) -> List[RegistryWorkflow]:
+        result = []
+        for w in self.get_workflows_metadata(user):
+            result.append(RegistryWorkflow(self.registry, w['id'], w['attributes']['title']))
+        return result
+
+    def get_index_workflow(self, user: User, workflow_identifier: str) -> RegistryWorkflow:
+        try:
+            w = self.get_workflow_metadata(user, workflow_identifier)
+            return RegistryWorkflow(self.registry, w['id'], w['attributes']['title'],
+                                    latest_version=w['attributes']['version'],
+                                    versions=[_['version'] for _ in w['attributes']['versions']]) if w else None
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                raise EntityNotFoundException(WorkflowRegistry, entity_id=workflow_identifier)
+            raise LifeMonitorException(original_error=e)
+
     def get_external_link(self, external_id: str, version: str) -> str:
-        return f"{self.registry.uri}/workflows/{external_id}?version={version}"
+        version_param = '' if not version or version == 'latest' else f"?version={version}"
+        return f"{self.registry.uri}/workflows/{external_id}{version_param}"
 
     def get_rocrate_external_link(self, external_id: str, version: str) -> str:
         return f'{self.registry.uri}/workflows/{external_id}/ro_crate?version={version}'