diff --git a/README.md b/README.md index c64e2e0..d622cb4 100644 --- a/README.md +++ b/README.md @@ -90,3 +90,83 @@ Enqueue a download job of a specific archive: Check the current status of an enqueued job: curl -i -X "GET" http://localhost:8989/api/1.0/status/ + +Docker container +---------------- + +For testing purposes, you can also build a Docker container using the resources in the docker/ folder: + + # build and start Docker container + docker/up + +This will build and start a Docker container that runs a nginx proxy server which will listen to connections on ports +9898 and 9899 and forward traffic to the archive-verify service running internally. API calls to port 9898 are done as +described above, e.g.: + + # interact with archive-verify service on port 9898 + curl 127.0.0.1:9898/api/1.0/status/1 + +API calls to port 9899 emulate how calls to the service running on Uppmax is done (i.e., going through a gateway). The +first path element for these calls should be verify/, e.g.: + + # interact with archive-verify service on port 9899 + curl 127.0.0.1:9899/verify/api/1.0/status/1 + +The container log output can be followed: + + # follow the container log output (Ctrl+C to stop) + docker/log + +In addition, the archive-verify service in the container is running with the MockPdcClient enabled. In the container, +there are two folders that can be used for testing with the mock client, `test_1_archive` and `test_2_archive`, e.g.: + + # enque a download job of the test archive available in the container + curl \ + -X "POST" \ + -d '{"host": "test_host", "description": "my-descr", "archive": "test_1_archive"}' \ + http://localhost:9898/api/1.0/download + + # { + # "status": "pending", + # "job_id": "d7a26f2e-d410-4a9b-a308-973821d0a021", + # "link": "http://localhost:9898/api/1.0/status/d7a26f2e-d410-4a9b-a308-973821d0a021", + # "path": "data/test_host/runfolders/test_1_archive", + # "action": "download" + # } + + # check the status of the download job + curl \ + http://localhost:9898/api/1.0/status/d7a26f2e-d410-4a9b-a308-973821d0a021 + + # { + # "state": "done", + # "msg": "Job d7a26f2e-d410-4a9b-a308-973821d0a021 has returned with result: Successfully verified archive md5sums." + # } + + # enque a verify job of the test archive available in the container, emulating a call to the service on Uppmax + curl \ + -X "POST" \ + -d '{"host": "test_host", "description": "my-descr", "archive": "test_2_archive"}' \ + http://localhost:9899/verify/api/1.0/verify + + # { + # "status": "pending", + # "job_id": "0124bdcb-7f31-4402-9251-ae766306ad49", + # "link": "http://localhost:9899/api/1.0/status/0124bdcb-7f31-4402-9251-ae766306ad49", + # "path": "data/test_host/runfolders/test_2_archive", + # "action": "verify" + # } + + # check the status of the download job + curl \ + http://localhost:9899/verify/api/1.0/status/0124bdcb-7f31-4402-9251-ae766306ad49 + + # { + # "state": "done", + # "msg": "Job 0124bdcb-7f31-4402-9251-ae766306ad49 has returned with result: Successfully verified archive md5sums." + # } + +The docker container can be stopped and removed: + + # stop and remove the running docker container + docker/down diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..0f1717a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.9-slim + +COPY . /archive-verify +WORKDIR /archive-verify + +# install services +RUN \ + apt-get update && \ + apt-get install -y git redis gcc nginx nano && \ + cp docker/nginx.conf /etc/nginx/nginx.conf && \ + rm -rf .venv && \ + python3 -m venv --upgrade-deps .venv && \ + .venv/bin/pip install -e .[test] + +RUN \ + .venv/bin/nosetests tests/ + +# setup test data +RUN \ + mkdir -p data/test_host/runfolders && \ + mkdir -p logs && \ + mkdir -p data/verify/test_1_archive/test_1_archive && \ + mkdir -p data/verify/test_2_archive/test_2_archive && \ + cd data/verify/test_1_archive/test_1_archive && \ + dd if=/dev/urandom of=test_1_data count=4000 bs=1024 && \ + md5sum test_1_data > checksums_prior_to_pdc.md5 && \ + cd ../../test_2_archive/test_2_archive && \ + dd if=/dev/urandom of=test_2_data count=4000 bs=1024 && \ + md5sum test_2_data > checksums_prior_to_pdc.md5 && \ + cd ../../../.. + +EXPOSE 9898 9899 + +CMD [ "docker/start.sh" ] diff --git a/docker/down b/docker/down new file mode 100755 index 0000000..b34450e --- /dev/null +++ b/docker/down @@ -0,0 +1,13 @@ +#! /bin/bash + +set -o errexit + +# get the container hash from file or error out if it doesn't exist +ID="$(cat docker/id.txt)" +rm -f docker/id.txt + +docker stop \ + "${ID}" + +docker rm \ + "${ID}" diff --git a/docker/log b/docker/log new file mode 100755 index 0000000..fa900a8 --- /dev/null +++ b/docker/log @@ -0,0 +1,9 @@ +#! /bin/bash + +set -o errexit + +# get the container hash from file or error out if it doesn't exist +ID="$(cat docker/id.txt)" + +docker logs \ + -f "${ID}" diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..dbd779b --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,22 @@ +events {} +http { + access_log /var/log/nginx/access.log; + server { + listen 9898; + access_log /var/log/nginx/local_access.log; + location / { + proxy_pass http://localhost:8989; + sub_filter_types application/json; + sub_filter 'http://localhost:8989' 'http://$host:$server_port'; + } + } + server { + listen 9899; + access_log /var/log/nginx/miarka_access.log; + location /verify/api/1.0/ { + proxy_pass http://localhost:8989/api/1.0/; + sub_filter_types application/json; + sub_filter 'http://localhost:8989' 'http://$host:$server_port'; + } + } +} diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 0000000..caf370e --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +redis-server & +/archive-verify/.venv/bin/rq worker & +/archive-verify/.venv/bin/archive-verify-ws -c=/archive-verify/config/ & +nginx & +wait diff --git a/docker/up b/docker/up new file mode 100755 index 0000000..edea6b4 --- /dev/null +++ b/docker/up @@ -0,0 +1,19 @@ +#! /bin/bash + +set -o errexit + +# build docker container +docker build \ + -t archive-verify:latest \ + -f docker/Dockerfile \ + . + +# start the container and store the container hash +ID="$(docker run \ + -d \ + -p 127.0.0.1:9898:9898 \ + -p 127.0.0.1:9899:9899 \ + archive-verify:latest)" + +# write the container hash to a file +echo "$ID" > docker/id.txt