Merge pull request #32 from sapporo-wes/introduce-workflow-run-crate

Introduce workflow run crate

.dockerignore

@@ -6,6 +6,6 @@
 !README_ja.md
 !README.md
 !sapporo
-!sapporo-wes-*
+!sapporo-wes-1-0-1-openapi-spec.yml
 !setup.py
 !uwsgi.yml

.github/workflows/deploy.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build_and_deploy_wheel:
     if: github.repository == 'sapporo-wes/sapporo-service'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: "Checkout"
         uses: actions/checkout@v3
@@ -48,7 +48,7 @@ jobs:
 
   deploy_ghcr:
     if: github.repository == 'sapporo-wes/sapporo-service'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: "Checkout"
         uses: actions/checkout@v3
@@ -81,7 +81,7 @@ jobs:
   create_release:
     if: github.repository == 'sapporo-wes/sapporo-service'
     needs: [build_and_deploy_wheel]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: "Download sdist"
         uses: actions/download-artifact@v2

.github/workflows/flake8.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   flake8:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/isort.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   isort:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/mypy.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   mypy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/pytest.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   pytest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v3

Dockerfile

@@ -1,18 +1,17 @@
-FROM python:3.8.13-buster as builder
+FROM python:3.8.17-bullseye as builder
 
 WORKDIR /app
 COPY . .
 RUN python3 -m pip install --no-cache-dir --progress-bar off -U pip setuptools wheel && \
     python3 -m pip install --no-cache-dir --progress-bar off .
 
-FROM python:3.8.13-slim-buster
+FROM python:3.8.17-slim-bullseye
 
 LABEL org.opencontainers.image.authors="Bioinformatics and DDBJ Center <t.ohta@nig.ac.jp>"
 LABEL org.opencontainers.image.url="https://github.com/sapporo-wes/sapporo-service"
 LABEL org.opencontainers.image.source="https://github.com/sapporo-wes/sapporo-service/blob/main/Dockerfile"
 LABEL org.opencontainers.image.version="1.4.9"
-LABEL org.opencontainers.image.description="sapporo-service is a standard implementation conforming to the \
-    Global Alliance for Genomics and Health (GA4GH) Workflow Execution Service (WES) API specification."
+LABEL org.opencontainers.image.description="sapporo-service is a standard implementation conforming to the Global Alliance for Genomics and Health (GA4GH) Workflow Execution Service (WES) API specification."
 LABEL org.opencontainers.image.licenses="Apache2.0"
 
 RUN apt update && \
@@ -26,10 +25,10 @@ RUN apt update && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /tmp
-RUN curl -O https://download.docker.com/linux/static/stable/$(uname -m)/docker-20.10.9.tgz && \
-    tar -xzf docker-20.10.9.tgz && \
+RUN curl -O https://download.docker.com/linux/static/stable/$(uname -m)/docker-23.0.6.tgz && \
+    tar -xzf docker-23.0.6.tgz && \
     mv docker/* /usr/bin/ && \
-    rm -rf docker docker-20.10.9.tgz
+    rm -rf docker docker-23.0.6.tgz
 
 COPY --from=builder /usr/local/lib/python3.8/site-packages /usr/local/lib/python3.8/site-packages
 COPY --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi

Dockerfile-dev

@@ -1,4 +1,4 @@
-FROM python:3.8.13-buster
+FROM python:3.8.17-bullseye
 
 RUN apt update && \
     apt install -y --no-install-recommends \
@@ -9,10 +9,10 @@ RUN apt update && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /tmp
-RUN curl -O https://download.docker.com/linux/static/stable/$(uname -m)/docker-20.10.9.tgz && \
-    tar -xzf docker-20.10.9.tgz && \
+RUN curl -O https://download.docker.com/linux/static/stable/$(uname -m)/docker-23.0.6.tgz && \
+    tar -xzf docker-23.0.6.tgz && \
     mv docker/* /usr/bin/ && \
-    rm -rf docker docker-20.10.9.tgz
+    rm -rf docker docker-23.0.6.tgz
 
 WORKDIR /app
 COPY . .

MANIFEST.in

@@ -1,3 +1,10 @@
 include LICENSE
-include sapporo-wes-spec.swagger.yml
+include README_ja.md
+include sapporo-wes-1-0-1-openapi-spec.yml
+include sapporo/executable_workflows.json
+include sapporo/executable_workflows.schema.json
+include sapporo/ro-terms.csv
+include sapporo/run.sh
+include sapporo/service-info.json
+include sapporo/service-info.schema.json
 include uwsgi.yml

dockerfile-for-wf-engines/Dockerfile-cromwell

@@ -0,0 +1,9 @@
+FROM broadinstitute/cromwell:80
+
+# install docker-ce
+RUN apt update && \
+    apt install -y apt-transport-https ca-certificates curl software-properties-common && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" && \
+    apt update && \
+    apt install -y docker-ce

dockerfile-for-wf-engines/README.md

@@ -0,0 +1,31 @@
+# Dockerfile for Workflow Engines
+
+## Why?
+
+When using the official docker image such as `broadinstitute/cromwell:85`,
+
+```
+docker: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.32' not found (required by docker)
+```
+
+an error like the above appears.
+
+In Sapporo, within `run.sh`,
+
+```bash
+local container="broadinstitute/cromwell:85"
+docker run --rm ${D_SOCK} -v ${run_dir}:${run_dir} -v /tmp:/tmp -v /usr/bin/docker:/usr/bin/docker -w=${exe_dir} ${container} run ${wf_engine_params} ${wf_url} -i ${wf_params} -m ${exe_dir}/metadata.json
+```
+
+We mount the docker command and docker socket on the host side and use them, but because the glibc version required by Docker on the host side and the glibc provided by the container image are different, an error like the one above occurs.
+
+Therefore, we create, provide, and use an workflow engine image with docker-ce installed, using the official image of the workflow engine as the base.
+
+## Cromwell
+
+To build and push:
+
+```bash
+$ docker build -f ./Dockerfile-cromwell -t ghcr.io/sapporo-wes/cromwell-with-docker:80 .
+$ docker push ghcr.io/sapporo-wes/cromwell-with-docker:80
+```

release.sh

@@ -30,10 +30,10 @@ echo "Rewrite files."
 sed -i "s/version=\"$PREV_VERSION\"/version=\"$NEW_VERSION\"/g" Dockerfile
 sed -i "s/sapporo-service:$PREV_VERSION/sapporo-service:$NEW_VERSION/g" docker-compose.yml
 sed -i "s/\"sapporo-version\": \"$PREV_VERSION\"/\"sapporo-version\": \"$NEW_VERSION\"/g" sapporo/service-info.json
-sed -i "s/version=\"$PREV_VERSION\"/version=\"$NEW_VERSION\"/g" setup.py
+sed -i "s/__version__ = \"$PREV_VERSION\"/__version__ = \"$NEW_VERSION\"/g" sapporo/__init__.py
 
 echo "Commit and push."
-git add Dockerfile docker-compose.yml sapporo/service-info.json setup.py
+git add Dockerfile docker-compose.yml sapporo/service-info.json sapporo/__init__.py
 git commit -m "Update version to $NEW_VERSION"
 git push origin main
 

sapporo/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+# coding: utf-8
+__version__ = "1.4.9"