From d2083d51ed47dd8622647fed32ee94b2248c674b Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 23 Feb 2024 15:28:44 +0100 Subject: [PATCH 01/20] Start with deployments --- deploy/Dockerfile.bartenderhaddock3 | 7 + deploy/Dockerfile.openssl | 6 + deploy/README.md | 7 + deploy/arq/bartender-config.yaml | 128 ++++++++++++++ deploy/arq/docker-compose.yml | 158 ++++++++++++++++++ .../docker-compose.dev.yml | 0 .../docker-compose.yml | 0 package.json | 6 +- 8 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 deploy/Dockerfile.bartenderhaddock3 create mode 100644 deploy/Dockerfile.openssl create mode 100644 deploy/README.md create mode 100644 deploy/arq/bartender-config.yaml create mode 100644 deploy/arq/docker-compose.yml rename docker-compose.dev.yml => deploy/docker-compose.dev.yml (100%) rename docker-compose.yml => deploy/docker-compose.yml (100%) diff --git a/deploy/Dockerfile.bartenderhaddock3 b/deploy/Dockerfile.bartenderhaddock3 new file mode 100644 index 00000000..da7f5ede --- /dev/null +++ b/deploy/Dockerfile.bartenderhaddock3 @@ -0,0 +1,7 @@ +# TODO get image with haddock3 from somewhere +FROM ghcr.io/haddocking/haddock3:latest as haddock3 + +# TODO publish image +FROM ghcr.io/ivresse/bartender:latest + +COPY --from=haddock3 /usr/local/bin/haddock3 /usr/local/bin/haddock3 diff --git a/deploy/Dockerfile.openssl b/deploy/Dockerfile.openssl new file mode 100644 index 00000000..ef4cbe8c --- /dev/null +++ b/deploy/Dockerfile.openssl @@ -0,0 +1,6 @@ +FROM alpine:3.19 + +RUN apk add --no-cache openssl + +CMD (test -e private_key.pem || openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048) && \ + (test -e public_key.pem || openssl rsa -pubout -in private_key.pem -out public_key.pem) diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..9cbc6b78 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,7 @@ +# Deployments + +## Deploying with minimal Docker container + +You want to run the webapp, bartender and haddock3 cli outside containers, but databases inside containers. +This is useful for development and debugging. + diff --git a/deploy/arq/bartender-config.yaml b/deploy/arq/bartender-config.yaml new file mode 100644 index 00000000..3472f6cc --- /dev/null +++ b/deploy/arq/bartender-config.yaml @@ -0,0 +1,128 @@ +# Example configuration file for haddock3 and haddock3-re. +# Expects executables to be in PATH, +# if not edit this file to use absolute path to executables +job_root_dir: jobs +destination_picker: bartender.picker:pick_first +applications: + haddock3: + command_template: haddock3 workflow.cfg + upload_needs: + - workflow.cfg + runimport: + summary: Import a HADDOCK3 run. + description: > + Upload an archive of haddock3 output. + The archive should have run dir as root. + The run should have haddock3-clean and haddock3-analyse executed on it. + command_template: mkdir -p output && mv * output || true +destinations: + local: + scheduler: + type: arq + redis_dsn: redis://redis:6379 + max_jobs: 1 + job_timeout: PT12H # 12 hours + filesystem: + type: local +interactive_applications: + rescore: + command_template: > + haddock3-re score + --w_elec {{w_elec|q}} --w_vdw {{w_vdw|q}} --w_desolv {{w_desolv|q}} --w_bsa {{w_bsa|q}} --w_air {{w_air|q}} + {{ capri_dir|q }} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Rescore a HADDOCK run with different weights. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + capri_dir: + type: string + w_air: + type: number + w_bsa: + type: number + w_desolv: + type: number + w_elec: + type: number + w_vdw: + type: number + required: + - module_nr + - capri_dir + - w_elec + - w_vdw + - w_desolv + - w_bsa + - w_air + type: object + reclustrmsd: + command_template: > + haddock3-re clustrmsd + {% if criterion == 'maxclust' -%} + --n_clusters {{n_clusters|q}} + {% else -%} + --clust_cutoff {{clust_cutoff|q}} + {% endif -%} + {% if min_population -%} + --min_population {{min_population|q}} + {% endif -%} + {{clustrmsd_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with RSMD and different parameters. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustrmsd_dir: + type: string + criterion: + type: string + enum: [maxclust, distance] + clust_cutoff: + type: number + n_clusters: + type: number + min_population: + type: number + required: + - module_nr + - criterion + - clustrmsd_dir + type: object + reclustfcc: + command_template: > + haddock3-re clustfcc + --clust_cutoff {{clust_cutoff|q}} --strictness {{strictness|q}} --min_population {{min_population|q}} + {{clustfcc_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with FCC and different parameters. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustfcc_dir: + type: string + clust_cutoff: + type: number + strictness: + type: number + min_population: + type: number + required: + - module_nr + - clustfcc_dir + - clust_cutoff + - strictness + - min_population + type: object diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml new file mode 100644 index 00000000..7e1b50c7 --- /dev/null +++ b/deploy/arq/docker-compose.yml @@ -0,0 +1,158 @@ +version: "3.9" + +services: + webapp: + # // TODO publish to ghcr.io + image: i-vresse/haddock3-webapp + build: + context: . + restart: always + depends_on: + bartender: + condition: service_started + webappdb: + condition: service_healthy + ports: + - "8080:8080" + env_file: + - ./.env + environment: + BARTENDER_API_URL: "http://bartender:8000" + DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres + BARTENDER_PRIVATE_KEY: /certs/private_key.pem + volumes: + - bartender-certs/private_key.pem:/certs/private_key.pem:ro + + webappdb: + image: postgres:15.4-bullseye + restart: always + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + volumes: + - webapp-db-data:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres + interval: 2s + timeout: 3s + retries: 40 + + webappmigrator: + build: + context: . + dockerfile: prisma/Dockerfile + # TODO use web app image or publish as seperate image to ghcr.io + command: npm run setup + restart: "no" + environment: + DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres + depends_on: + webappdb: + condition: service_healthy + + bartender: + build: + context: . + dockerfile: ./deploy/Dockerfile.bartenderhaddock3 + image: bartender:${BARTENDER_VERSION:-latest} + restart: always + env_file: + - ../bartender/.env + volumes: + - type: bind + source: ./bartender-config.yaml + target: /app/src/config.yaml + # TODO make sure certs are available + - bartender-certs/private_key.pem:/certs/private_key.pem:ro + - bartender-job-data:/tmp/jobs + depends_on: + bartenderdb: + condition: service_healthy + redis: + condition: service_started + environment: + BARTENDER_HOST: 0.0.0.0 + BARTENDER_DB_HOST: bartender-db + BARTENDER_DB_PORT: 5432 + BARTENDER_DB_USER: bartender + BARTENDER_DB_PASS: bartender + BARTENDER_DB_BASE: bartender + BARTENDER_JOB_ROOT: /tmp/jobs + BARTENDER_PUBLIC_KEY: /certs/public_key.pem + + # TODO user same database for webapp and bartender + # the tables do not overlap + bartenderdb: + image: postgres:15.4-bullseye + hostname: bartender-db + environment: + POSTGRES_PASSWORD: "bartender" + POSTGRES_USER: "bartender" + POSTGRES_DB: "bartender" + volumes: + - bartender-db-data:/var/lib/postgresql/data + restart: always + healthcheck: + test: pg_isready -U bartender + interval: 2s + timeout: 3s + retries: 40 + + bartendermigrator: + build: + context: . + dockerfile: ./deploy/Dockerfile.bartenderhaddock3 + image: bartender:${BARTENDER_VERSION:-latest} + restart: "no" + command: alembic upgrade head + environment: + BARTENDER_DB_HOST: bartender-db + BARTENDER_DB_PORT: 5432 + BARTENDER_DB_USER: bartender + BARTENDER_DB_PASS: bartender + BARTENDER_DB_BASE: bartender + depends_on: + bartenderdb: + condition: service_healthy + + redis: + image: redis:6.2.6 + restart: always + ports: + - "6379:6379" + + bartenderworker: + build: + context: . + dockerfile: ./deploy/Dockerfile.bartenderhaddock3 + image: bartender:${BARTENDER_VERSION:-latest} + restart: always + command: bartender perform + env_file: + - ../bartender/.env + volumes: + - type: bind + source: ./bartender-config.yaml + target: /app/src/config.yaml + - bartender-job-data:/tmp/jobs + depends_on: + redis: + condition: service_started + environment: + BARTENDER_JOB_ROOT: /tmp/jobs + + certmaker: + build: + context: . + dockerfile: ./deploy/Dockerfile.openssl + volumes: + - bartender-certs:/certs + working_dir: /certs + restart: "no" + +volumes: + webapp-db-data: + bartender-db-data: + bartender-job-data: + bartender-certs: \ No newline at end of file diff --git a/docker-compose.dev.yml b/deploy/docker-compose.dev.yml similarity index 100% rename from docker-compose.dev.yml rename to deploy/docker-compose.dev.yml diff --git a/docker-compose.yml b/deploy/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to deploy/docker-compose.yml diff --git a/package.json b/package.json index fe9ddea0..c71680d9 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,9 @@ "test": "vitest", "typecheck": "tsc", "setup": "prisma generate && prisma migrate deploy && prisma db seed", - "docker:dev": "docker compose -f docker-compose.dev.yml up", - "docker:devrm": "docker compose -f docker-compose.dev.yml rm -v", - "psql:dev": "docker compose -f docker-compose.dev.yml exec userdb psql -U postgres -d postgres" + "docker:dev": "docker compose -f deploy/docker-compose.dev.yml up", + "docker:devrm": "docker compose -f deploy/docker-compose.dev.yml rm -v", + "psql:dev": "docker compose -f deploy/docker-compose.dev.yml exec userdb psql -U postgres -d postgres" }, "prettier": {}, "eslintIgnore": [ From 8ab469be373254fe3b28c62a66a760a786f53d25 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 23 Feb 2024 15:38:18 +0100 Subject: [PATCH 02/20] Setup db in same container as webapp/webservice --- deploy/arq/docker-compose.yml | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 7e1b50c7..0e21009e 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -7,6 +7,8 @@ services: build: context: . restart: always + command: + npm run setup && npm run start depends_on: bartender: condition: service_started @@ -38,25 +40,14 @@ services: timeout: 3s retries: 40 - webappmigrator: - build: - context: . - dockerfile: prisma/Dockerfile - # TODO use web app image or publish as seperate image to ghcr.io - command: npm run setup - restart: "no" - environment: - DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres - depends_on: - webappdb: - condition: service_healthy - bartender: build: context: . dockerfile: ./deploy/Dockerfile.bartenderhaddock3 image: bartender:${BARTENDER_VERSION:-latest} restart: always + command: + alembic upgrade head && bartender serve env_file: - ../bartender/.env volumes: @@ -99,23 +90,6 @@ services: timeout: 3s retries: 40 - bartendermigrator: - build: - context: . - dockerfile: ./deploy/Dockerfile.bartenderhaddock3 - image: bartender:${BARTENDER_VERSION:-latest} - restart: "no" - command: alembic upgrade head - environment: - BARTENDER_DB_HOST: bartender-db - BARTENDER_DB_PORT: 5432 - BARTENDER_DB_USER: bartender - BARTENDER_DB_PASS: bartender - BARTENDER_DB_BASE: bartender - depends_on: - bartenderdb: - condition: service_healthy - redis: image: redis:6.2.6 restart: always From 46db988e6510001ba739614c5835e39d63d3eaaa Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 23 Feb 2024 15:52:20 +0100 Subject: [PATCH 03/20] use docker-compose-wait to wait for appearance of ssl keys. --- Dockerfile | 2 ++ deploy/Dockerfile.bartenderhaddock3 | 8 ++++++-- deploy/arq/docker-compose.yml | 11 ++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8bfdbee8..8357350a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,4 +49,6 @@ COPY --from=build /myapp/build /myapp/build COPY --from=build /myapp/public /myapp/public COPY --from=build /myapp/package.json /myapp/package.json +COPY --from=ghcr.io/ufoscout/docker-compose-wait:latest /wait /wait + ENTRYPOINT [ "npm", "run", "start" ] \ No newline at end of file diff --git a/deploy/Dockerfile.bartenderhaddock3 b/deploy/Dockerfile.bartenderhaddock3 index da7f5ede..b81986d5 100644 --- a/deploy/Dockerfile.bartenderhaddock3 +++ b/deploy/Dockerfile.bartenderhaddock3 @@ -1,7 +1,11 @@ # TODO get image with haddock3 from somewhere -FROM ghcr.io/haddocking/haddock3:latest as haddock3 +ARG HADDOCK3_VERSION=latest +FROM ghcr.io/haddocking/haddock3:${HADDOCK3_VERSION} as haddock3 + +COPY --from=ghcr.io/ufoscout/docker-compose-wait:latest /wait /wait # TODO publish image -FROM ghcr.io/ivresse/bartender:latest +ARG BARTENDER_VERSION=latest +FROM ghcr.io/ivresse/bartender:${BARTENDER_VERSION} COPY --from=haddock3 /usr/local/bin/haddock3 /usr/local/bin/haddock3 diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 0e21009e..d118feec 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -2,13 +2,13 @@ version: "3.9" services: webapp: - # // TODO publish to ghcr.io + # TODO publish to ghcr.io image: i-vresse/haddock3-webapp build: context: . restart: always command: - npm run setup && npm run start + /wait && npm run setup && npm run start depends_on: bartender: condition: service_started @@ -22,6 +22,7 @@ services: BARTENDER_API_URL: "http://bartender:8000" DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres BARTENDER_PRIVATE_KEY: /certs/private_key.pem + WAIT_PATHS: /certs/private_key.pem volumes: - bartender-certs/private_key.pem:/certs/private_key.pem:ro @@ -47,15 +48,14 @@ services: image: bartender:${BARTENDER_VERSION:-latest} restart: always command: - alembic upgrade head && bartender serve + /wait && alembic upgrade head && bartender serve env_file: - ../bartender/.env volumes: - type: bind source: ./bartender-config.yaml target: /app/src/config.yaml - # TODO make sure certs are available - - bartender-certs/private_key.pem:/certs/private_key.pem:ro + - bartender-certs/public_key.pem:/certs/public_key.pem:ro - bartender-job-data:/tmp/jobs depends_on: bartenderdb: @@ -71,6 +71,7 @@ services: BARTENDER_DB_BASE: bartender BARTENDER_JOB_ROOT: /tmp/jobs BARTENDER_PUBLIC_KEY: /certs/public_key.pem + WAIT_PATHS: /certs/private_key.pem # TODO user same database for webapp and bartender # the tables do not overlap From 2a63fb5abad33bc2b7c2915828df656008715fd0 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 23 Feb 2024 15:54:18 +0100 Subject: [PATCH 04/20] More todos --- deploy/Dockerfile.bartenderhaddock3 | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/Dockerfile.bartenderhaddock3 b/deploy/Dockerfile.bartenderhaddock3 index b81986d5..42a73331 100644 --- a/deploy/Dockerfile.bartenderhaddock3 +++ b/deploy/Dockerfile.bartenderhaddock3 @@ -1,4 +1,5 @@ # TODO get image with haddock3 from somewhere +# TODO have image contain cns executable, likely from private registry, which would require `docker login` before building. ARG HADDOCK3_VERSION=latest FROM ghcr.io/haddocking/haddock3:${HADDOCK3_VERSION} as haddock3 From 7eff9df01df00dab4dfa570fc1c54fd6e9107fd1 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 23 Feb 2024 15:54:40 +0100 Subject: [PATCH 05/20] FOrmat --- deploy/README.md | 1 - deploy/arq/bartender-config.yaml | 34 ++++++++++++++++---------------- deploy/arq/docker-compose.yml | 10 ++++------ 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 9cbc6b78..4c3ecdbe 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -4,4 +4,3 @@ You want to run the webapp, bartender and haddock3 cli outside containers, but databases inside containers. This is useful for development and debugging. - diff --git a/deploy/arq/bartender-config.yaml b/deploy/arq/bartender-config.yaml index 3472f6cc..1e2d435a 100644 --- a/deploy/arq/bartender-config.yaml +++ b/deploy/arq/bartender-config.yaml @@ -7,7 +7,7 @@ applications: haddock3: command_template: haddock3 workflow.cfg upload_needs: - - workflow.cfg + - workflow.cfg runimport: summary: Import a HADDOCK3 run. description: > @@ -21,7 +21,7 @@ destinations: type: arq redis_dsn: redis://redis:6379 max_jobs: 1 - job_timeout: PT12H # 12 hours + job_timeout: PT12H # 12 hours filesystem: type: local interactive_applications: @@ -52,13 +52,13 @@ interactive_applications: w_vdw: type: number required: - - module_nr - - capri_dir - - w_elec - - w_vdw - - w_desolv - - w_bsa - - w_air + - module_nr + - capri_dir + - w_elec + - w_vdw + - w_desolv + - w_bsa + - w_air type: object reclustrmsd: command_template: > @@ -93,9 +93,9 @@ interactive_applications: min_population: type: number required: - - module_nr - - criterion - - clustrmsd_dir + - module_nr + - criterion + - clustrmsd_dir type: object reclustfcc: command_template: > @@ -120,9 +120,9 @@ interactive_applications: min_population: type: number required: - - module_nr - - clustfcc_dir - - clust_cutoff - - strictness - - min_population + - module_nr + - clustfcc_dir + - clust_cutoff + - strictness + - min_population type: object diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index d118feec..916e2f64 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -7,8 +7,7 @@ services: build: context: . restart: always - command: - /wait && npm run setup && npm run start + command: /wait && npm run setup && npm run start depends_on: bartender: condition: service_started @@ -47,8 +46,7 @@ services: dockerfile: ./deploy/Dockerfile.bartenderhaddock3 image: bartender:${BARTENDER_VERSION:-latest} restart: always - command: - /wait && alembic upgrade head && bartender serve + command: /wait && alembic upgrade head && bartender serve env_file: - ../bartender/.env volumes: @@ -96,7 +94,7 @@ services: restart: always ports: - "6379:6379" - + bartenderworker: build: context: . @@ -130,4 +128,4 @@ volumes: webapp-db-data: bartender-db-data: bartender-job-data: - bartender-certs: \ No newline at end of file + bartender-certs: From d1aad851c2ab40214ce7dfcda372a762517f2090 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 10:56:41 +0100 Subject: [PATCH 06/20] Move openapi-fetch to non-dev deps --- package-lock.json | 6 ++---- package.json | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index e4d59c22..fe6d79d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "js-yaml": "^4.1.0", "jszip": "^3.10.1", "lucide-react": "^0.321.0", + "openapi-fetch": "^0.8.2", "plotly.js": "^2.26.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -72,7 +73,6 @@ "eslint-plugin-react": "^7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "happy-dom": "^9.20.3", - "openapi-fetch": "^0.8.2", "openapi-typescript": "^7.0.0-next.5", "prettier": "^2.8.7", "prettier-plugin-tailwindcss": "^0.2.7", @@ -12033,7 +12033,6 @@ "version": "0.8.2", "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.8.2.tgz", "integrity": "sha512-4g+NLK8FmQ51RW6zLcCBOVy/lwYmFJiiT+ckYZxJWxUxH4XFhsNcX2eeqVMfVOi+mDNFja6qDXIZAz2c5J/RVw==", - "dev": true, "dependencies": { "openapi-typescript-helpers": "^0.0.5" } @@ -12059,8 +12058,7 @@ "node_modules/openapi-typescript-helpers": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.5.tgz", - "integrity": "sha512-MRffg93t0hgGZbYTxg60hkRIK2sRuEOHEtCUgMuLgbCC33TMQ68AmxskzUlauzZYD47+ENeGV/ElI7qnWqrAxA==", - "dev": true + "integrity": "sha512-MRffg93t0hgGZbYTxg60hkRIK2sRuEOHEtCUgMuLgbCC33TMQ68AmxskzUlauzZYD47+ENeGV/ElI7qnWqrAxA==" }, "node_modules/openapi-typescript/node_modules/supports-color": { "version": "9.4.0", diff --git a/package.json b/package.json index c71680d9..e4648d76 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "js-yaml": "^4.1.0", "jszip": "^3.10.1", "lucide-react": "^0.321.0", + "openapi-fetch": "^0.8.2", "plotly.js": "^2.26.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -95,7 +96,6 @@ "eslint-plugin-react": "^7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "happy-dom": "^9.20.3", - "openapi-fetch": "^0.8.2", "openapi-typescript": "^7.0.0-next.5", "prettier": "^2.8.7", "prettier-plugin-tailwindcss": "^0.2.7", From a8ca08e550c2a5df057e8ceccd8ae1e48552424f Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 14:04:29 +0100 Subject: [PATCH 07/20] Add SESSIONS_SECRET_FILE env var support --- app/session.server.ts | 7 ++++++- docs/auth.md | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/session.server.ts b/app/session.server.ts index ea6971d9..2c51be86 100644 --- a/app/session.server.ts +++ b/app/session.server.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs' import { createCookieSessionStorage } from "@remix-run/node"; import { createThemeSessionResolver } from "remix-themes"; @@ -10,7 +11,11 @@ export const sessionStorage = createCookieSessionStorage({ path: "/", sameSite: "lax", maxAge: 604_800, // one week - secrets: [process.env.SESSION_SECRET || "somebadsecret"], + secrets: [ + process.env.SESSION_SECRET || + (process.env.SESSION_SECRET_FILE && readFileSync(process.env.SESSION_SECRET_FILE).toString()) || + "somebadsecret" + ], secure: process.env.NODE_ENV === "production", }, }); diff --git a/docs/auth.md b/docs/auth.md index ed02e182..03d8c59d 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -24,6 +24,8 @@ SESSION_SECRET=... A random secret string can be generated with `openssl rand -base64 32`. +Or use `SESSIONS_SECRET_FILE` environment variable to read the secret from a file. + ## Social logins To enable GitHub or Orcid or EGI Check-in login the web app needs following environment variables. From 5b8945d3c4efcef021e04dacc0150db16ab0d708 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 14:20:22 +0100 Subject: [PATCH 08/20] `docker compose -f deploy/arq/docker-compose.yml up --build` works --- .gitignore | 3 +- README.md | 14 ++-- Dockerfile => deploy/Dockerfile | 12 ++- deploy/Dockerfile.bartenderhaddock3 | 79 ++++++++++++++++-- deploy/Dockerfile.openssl | 3 +- deploy/README.md | 18 +++- deploy/arq/.env | 5 ++ deploy/arq/README.md | 17 ++++ deploy/arq/bartender-config.yaml | 7 +- deploy/arq/docker-compose.yml | 75 +++++++---------- deploy/docker-compose.yml | 124 ---------------------------- docs/docker.md | 26 ------ 12 files changed, 164 insertions(+), 219 deletions(-) rename Dockerfile => deploy/Dockerfile (79%) create mode 100644 deploy/arq/.env create mode 100644 deploy/arq/README.md delete mode 100644 deploy/docker-compose.yml delete mode 100644 docs/docker.md diff --git a/.gitignore b/.gitignore index fad0d964..ecbd5db6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ node_modules /.cache /build /public/build -.env +/.env .vscode/ # openapi-generator-cli bookkeeping @@ -18,3 +18,4 @@ node_modules /public_key.pem Caddyfile +/deploy/cns diff --git a/README.md b/README.md index a1406952..2c10bf70 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,6 @@ npm run setup (You can reset database with `npx prisma migrate reset`.) -The database setup should be run only once for a fresh database. -Whenever you change the `prisma/schema.prisma` file you need to - -1. Use [prisma migrate](https://www.prisma.io/docs/concepts/components/prisma-migrate) to generate a migration and to update the database. -2. Run `npx prisma generate` to generate the prisma client. - Start [remix](https://remix.run) development server from your terminal with: ```sh @@ -92,6 +86,12 @@ This will refresh & rebuild assets on file changes. ## Other development commands +The database setup should be run only once for a fresh database. +Whenever you change the `prisma/schema.prisma` file you need to + +1. Use [prisma migrate](https://www.prisma.io/docs/concepts/components/prisma-migrate) to generate a migration and to update the database. +2. Run `npx prisma generate` to generate the prisma client. + To format according to [prettier](https://prettier.io) run ```sh @@ -135,7 +135,7 @@ Then run the app in production mode: npm start ``` -The web application can be run inside a Docker container together with all its dependent containers, see [docs/docker.md](docs/docker.md). +Other deployments with containers are explained in [deploy/README.md](deploy/README.md). ## Stack diff --git a/Dockerfile b/deploy/Dockerfile similarity index 79% rename from Dockerfile rename to deploy/Dockerfile index 8357350a..f75d125d 100644 --- a/Dockerfile +++ b/deploy/Dockerfile @@ -1,9 +1,13 @@ +# Docker file for haddock3-webapp + # base node image -FROM node:18-bullseye-slim as base +FROM node:18-bookworm-slim as base # set for base and all layer that inherit from it ENV NODE_ENV production +RUN apt update && apt install -y libssl3 + # Install all node_modules, including dev dependencies FROM base as deps @@ -46,9 +50,13 @@ COPY --from=production-deps /myapp/node_modules /myapp/node_modules COPY --from=build /myapp/node_modules/.prisma /myapp/node_modules/.prisma COPY --from=build /myapp/build /myapp/build +COPY --from=build /myapp/prisma /myapp/prisma COPY --from=build /myapp/public /myapp/public + COPY --from=build /myapp/package.json /myapp/package.json COPY --from=ghcr.io/ufoscout/docker-compose-wait:latest /wait /wait -ENTRYPOINT [ "npm", "run", "start" ] \ No newline at end of file +ENTRYPOINT [ "npm", "run", "start" ] + +HEALTHCHECK --interval=30s --timeout=5s CMD curl --fail http://localhost:8080/ || exit 1 diff --git a/deploy/Dockerfile.bartenderhaddock3 b/deploy/Dockerfile.bartenderhaddock3 index 42a73331..53b2cd8a 100644 --- a/deploy/Dockerfile.bartenderhaddock3 +++ b/deploy/Dockerfile.bartenderhaddock3 @@ -1,12 +1,75 @@ -# TODO get image with haddock3 from somewhere -# TODO have image contain cns executable, likely from private registry, which would require `docker login` before building. -ARG HADDOCK3_VERSION=latest -FROM ghcr.io/haddocking/haddock3:${HADDOCK3_VERSION} as haddock3 +# Dockerfile for bartender web service with haddock3, lightdock, gdock executables. +FROM python:3.9-slim-bookworm + +ARG HADDOCK3_VERSION=main +ARG BARTENDER_VERSION=main +ARG GDOCK_VERSION=master +ARG LIGHTDOCK_VERSION=0.9.4 COPY --from=ghcr.io/ufoscout/docker-compose-wait:latest /wait /wait -# TODO publish image -ARG BARTENDER_VERSION=latest -FROM ghcr.io/ivresse/bartender:${BARTENDER_VERSION} +# Common dependencies ================================================================================================= + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential git ca-certificates curl libgfortran5 \ + wget \ + && \ + apt-get clean && rm -rf /var/lib/apt/lists/* \ + && \ + pip install --upgrade pip + +# haddock3 ============================================================================================================ +WORKDIR /opt/haddock3 +RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git checkout ${HADDOCK3_VERSION} +WORKDIR /opt/haddock3/src/fcc/src +RUN make +WORKDIR /opt/haddock3 +RUN pip install -r requirements.txt && python setup.py develop +COPY ./cns /opt/haddock3/bin/cns +WORKDIR / +ENV PYTHONPATH=/opt/haddock3/src + +# lightdock ========================================================================================================== + +RUN pip install lightdock==${LIGHTDOCK_VERSION} + +# gdock ============================================================================================================= + +WORKDIR /opt/gdock +RUN git clone https://github.com/gdocking/gdock.git . && git checkout ${GDOCK_VERSION} +RUN python setup.py develop +RUN bash install.sh /opt/gdock +ENV GDOCK_PATH=/opt/gdock + +# bartender ========================================================================================================== + +WORKDIR /opt/bartender +RUN git clone https://github.com/i-VRESSE/bartender.git . && git checkout ${BARTENDER_VERSION} +RUN pip install . + +CMD ["/usr/local/bin/bartender", "serve"] + +HEALTHCHECK --interval=30s --timeout=5s CMD curl --fail http://localhost:8000/api/health || exit 1 + +# non-root user ====================================================================================================== + +ARG USERNAME=bartender +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # + # [Optional] Add sudo support. Omit if you don't need to install software after connecting. + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +RUN mkdir /jobs && chown -R $USERNAME:$USERNAME /jobs -COPY --from=haddock3 /usr/local/bin/haddock3 /usr/local/bin/haddock3 +# All commands from here on will be run as the bartender user and not as root. +USER ${USERNAME} diff --git a/deploy/Dockerfile.openssl b/deploy/Dockerfile.openssl index ef4cbe8c..386fcea5 100644 --- a/deploy/Dockerfile.openssl +++ b/deploy/Dockerfile.openssl @@ -3,4 +3,5 @@ FROM alpine:3.19 RUN apk add --no-cache openssl CMD (test -e private_key.pem || openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048) && \ - (test -e public_key.pem || openssl rsa -pubout -in private_key.pem -out public_key.pem) + (test -e public_key.pem || openssl rsa -pubout -in private_key.pem -out public_key.pem) && \ + (test -e session.secret || openssl rand -hex 32 > session.secret) diff --git a/deploy/README.md b/deploy/README.md index 4c3ecdbe..5481d45d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,18 @@ # Deployments -## Deploying with minimal Docker container +> [!NOTE] +> All deployment methods require the `cns` executable to available in this directory. -You want to run the webapp, bartender and haddock3 cli outside containers, but databases inside containers. -This is useful for development and debugging. +## Methods + +* [Haddock3 webapp with single worker and docker compose](arq/README.md) + +## Versions + +The webapp version is what is currently checked out. + +Versions of bartender, haddock3, gdock and lightdock are defined in the start of the [./Dockerfile.bartenderhaddock3](./Dockerfile.bartenderhaddock3) file. They can be overwritten during build with + +```shell +docker compose -f build --build-arg HADDOCK3_VERSION=v3.0.0-beta.5 +``` diff --git a/deploy/arq/.env b/deploy/arq/.env new file mode 100644 index 00000000..8b6f54a1 --- /dev/null +++ b/deploy/arq/.env @@ -0,0 +1,5 @@ +# In the docker-compose.yml the session, database and bartender are configured. +# Here you can configure the haddock3 webapp for haddock3 and social logins +# See https://github.com/i-VRESSE/haddock3-webapp/blob/main/docs/rewrite.md +# and https://github.com/i-VRESSE/haddock3-webapp/blob/main/docs/auth.md#social-logins +# respectively for more information. diff --git a/deploy/arq/README.md b/deploy/arq/README.md new file mode 100644 index 00000000..39bc25ad --- /dev/null +++ b/deploy/arq/README.md @@ -0,0 +1,17 @@ +# Haddock3 webapp with single worker + +```shell +# Must be in root of repo +cd ../.. +# Need cns executable in deploy directory, so it can be copied into the Docker image +cp deploy/cns +docker compose -f deploy/arq/docker-compose.yml up --build +``` + +The haddock3 webapp should be running on http://localhost:8080 + +Next steps are to go to http://localhost:8080/register to register as admin, give your self an expertise level at http://localhost:8080/admin/users and finally submit a job. + +## Configuration + +The webapp can be configured using a [.env](.env). diff --git a/deploy/arq/bartender-config.yaml b/deploy/arq/bartender-config.yaml index 1e2d435a..53a703bf 100644 --- a/deploy/arq/bartender-config.yaml +++ b/deploy/arq/bartender-config.yaml @@ -1,7 +1,6 @@ -# Example configuration file for haddock3 and haddock3-re. -# Expects executables to be in PATH, -# if not edit this file to use absolute path to executables -job_root_dir: jobs +# This file is the configuration file for the bartender service. +# This configuration works for the webapp and the docker compose deployment. +job_root_dir: /jobs destination_picker: bartender.picker:pick_first applications: haddock3: diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 916e2f64..88340c85 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -2,12 +2,13 @@ version: "3.9" services: webapp: - # TODO publish to ghcr.io - image: i-vresse/haddock3-webapp + image: ghcr.io/i-vresse/haddock3-webapp build: - context: . + context: ../.. + dockerfile: deploy/Dockerfile restart: always - command: /wait && npm run setup && npm run start + entrypoint: /usr/bin/sh + command: ['-c', '/wait && npx prisma migrate deploy && npm run start'] depends_on: bartender: condition: service_started @@ -15,15 +16,14 @@ services: condition: service_healthy ports: - "8080:8080" - env_file: - - ./.env environment: BARTENDER_API_URL: "http://bartender:8000" DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres BARTENDER_PRIVATE_KEY: /certs/private_key.pem + SESSION_SECRET_FILE: /certs/session.secret WAIT_PATHS: /certs/private_key.pem volumes: - - bartender-certs/private_key.pem:/certs/private_key.pem:ro + - bartender-certs:/certs:ro webappdb: image: postgres:15.4-bullseye @@ -41,20 +41,18 @@ services: retries: 40 bartender: + image: ghcr.io/i-vresse/bartender build: - context: . - dockerfile: ./deploy/Dockerfile.bartenderhaddock3 - image: bartender:${BARTENDER_VERSION:-latest} + context: .. + dockerfile: ./Dockerfile.bartenderhaddock3 restart: always - command: /wait && alembic upgrade head && bartender serve - env_file: - - ../bartender/.env + command: sh -c '/wait && alembic upgrade head && bartender serve' volumes: - type: bind source: ./bartender-config.yaml - target: /app/src/config.yaml - - bartender-certs/public_key.pem:/certs/public_key.pem:ro - - bartender-job-data:/tmp/jobs + target: /opt/bartender/config.yaml + - bartender-certs:/certs:ro + - bartender-job-data:/jobs depends_on: bartenderdb: condition: service_healthy @@ -62,29 +60,25 @@ services: condition: service_started environment: BARTENDER_HOST: 0.0.0.0 - BARTENDER_DB_HOST: bartender-db + BARTENDER_DB_HOST: bartenderdb BARTENDER_DB_PORT: 5432 - BARTENDER_DB_USER: bartender - BARTENDER_DB_PASS: bartender - BARTENDER_DB_BASE: bartender - BARTENDER_JOB_ROOT: /tmp/jobs + BARTENDER_DB_USER: postgres + BARTENDER_DB_PASS: postgres + BARTENDER_DB_BASE: postgres BARTENDER_PUBLIC_KEY: /certs/public_key.pem WAIT_PATHS: /certs/private_key.pem - # TODO user same database for webapp and bartender - # the tables do not overlap bartenderdb: image: postgres:15.4-bullseye - hostname: bartender-db + restart: always environment: - POSTGRES_PASSWORD: "bartender" - POSTGRES_USER: "bartender" - POSTGRES_DB: "bartender" + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres volumes: - bartender-db-data:/var/lib/postgresql/data - restart: always healthcheck: - test: pg_isready -U bartender + test: pg_isready -U postgres interval: 2s timeout: 3s retries: 40 @@ -92,39 +86,34 @@ services: redis: image: redis:6.2.6 restart: always - ports: - - "6379:6379" bartenderworker: + image: ghcr.io/i-vresse/bartender build: - context: . - dockerfile: ./deploy/Dockerfile.bartenderhaddock3 - image: bartender:${BARTENDER_VERSION:-latest} + context: .. + dockerfile: ./Dockerfile.bartenderhaddock3 restart: always - command: bartender perform - env_file: - - ../bartender/.env + command: sh -c 'bartender perform' volumes: - type: bind source: ./bartender-config.yaml - target: /app/src/config.yaml - - bartender-job-data:/tmp/jobs + target: /opt/bartender/config.yaml + - bartender-job-data:/jobs depends_on: redis: condition: service_started - environment: - BARTENDER_JOB_ROOT: /tmp/jobs certmaker: build: - context: . - dockerfile: ./deploy/Dockerfile.openssl + context: .. + dockerfile: ./Dockerfile.openssl volumes: - bartender-certs:/certs working_dir: /certs restart: "no" volumes: + # Tried to use single postgres, but prisma complains about non-empty database (P3005 error). webapp-db-data: bartender-db-data: bartender-job-data: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml deleted file mode 100644 index c5923995..00000000 --- a/deploy/docker-compose.yml +++ /dev/null @@ -1,124 +0,0 @@ -version: "3.9" - -services: - webapp: - # // TODO publish to ghcr.io - image: i-vresse/haddock3-webapp - build: - context: . - restart: always - depends_on: - bartender: - condition: service_started - webappdb: - condition: service_healthy - ports: - - "8080:8080" - env_file: - - ./.env - environment: - BARTENDER_API_URL: "http://bartender:8000" - DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres - BARTENDER_PRIVATE_KEY: /app/src/private_key.pem - volumes: - - type: bind - source: ./private_key.pem - target: /app/src/private_key.pem - - webappdb: - image: postgres:15.4-bullseye - restart: always - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres - volumes: - - webapp-db-data:/var/lib/postgresql/data - healthcheck: - test: pg_isready -U postgres - interval: 2s - timeout: 3s - retries: 40 - - webappmigrator: - build: - context: . - dockerfile: prisma/Dockerfile - # TODO use web app image or publish as seperate image to ghcr.io - command: npm run setup - restart: "no" - environment: - DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres - depends_on: - webappdb: - condition: service_healthy - - bartender: - build: - context: ../bartender - dockerfile: ./deploy/Dockerfile - image: bartender:${BARTENDER_VERSION:-latest} - restart: always - env_file: - - ../bartender/.env - volumes: - - type: bind - source: ../bartender/config.yaml - target: /app/src/config.yaml - - type: bind - source: ./public_key.pem - target: /app/src/public_key.pem - # to store jobs on NFS share replace line below with bind mount - - bartender-job-data:/tmp/jobs - depends_on: - bartenderdb: - condition: service_healthy - environment: - BARTENDER_HOST: 0.0.0.0 - BARTENDER_DB_HOST: bartender-db - BARTENDER_DB_PORT: 5432 - BARTENDER_DB_USER: bartender - BARTENDER_DB_PASS: bartender - BARTENDER_DB_BASE: bartender - BARTENDER_JOB_ROOT: /tmp/jobs - BARTENDER_PUBLIC_KEY: /app/src/public_key.pem - - # TODO user same database for webapp and bartender - # the tables do not overlap - bartenderdb: - image: postgres:15.4-bullseye - hostname: bartender-db - environment: - POSTGRES_PASSWORD: "bartender" - POSTGRES_USER: "bartender" - POSTGRES_DB: "bartender" - volumes: - - bartender-db-data:/var/lib/postgresql/data - restart: always - healthcheck: - test: pg_isready -U bartender - interval: 2s - timeout: 3s - retries: 40 - - bartendermigrator: - build: - context: ../bartender - dockerfile: ./deploy/Dockerfile - image: bartender:${BARTENDER_VERSION:-latest} - restart: "no" - command: alembic upgrade head - environment: - BARTENDER_DB_HOST: bartender-db - BARTENDER_DB_PORT: 5432 - BARTENDER_DB_USER: bartender - BARTENDER_DB_PASS: bartender - BARTENDER_DB_BASE: bartender - depends_on: - bartenderdb: - condition: service_healthy - -volumes: - webapp-db-data: - bartender-db-data: - bartender-job-data: diff --git a/docs/docker.md b/docs/docker.md deleted file mode 100644 index f30bb83e..00000000 --- a/docs/docker.md +++ /dev/null @@ -1,26 +0,0 @@ -# Deployment with docker - -The web application can be run inside a Docker container together with all its dependent containers. - -Requirements: - -1. Private key `./private_key.pem` and public key `./public_key.pem`. -2. `./.env` file for haddock3 web application. -3. [bartender repo](https://github.com/i-VRESSE/bartender) to be cloned in `../bartender` directory. -4. bartender repo should have [.env file](https://github.com/i-VRESSE/bartender/blob/main/docs/configuration.md#environment-variables) -5. bartender repo should have a [config.yaml file](https://github.com/i-VRESSE/bartender/blob/main/docs/configuration.md#configuration-file) - 1. The `job_root_dir` key should be set to `/tmp/jobs` - -Build with - -```sh -docker compose build -``` - -Run with - -```sh -docker compose up -``` - -Web application running at http://localhost:8080 . From 954d9f23b69d21176f2894d70665294e9864ea0b Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 14:24:23 +0100 Subject: [PATCH 09/20] Format --- app/session.server.ts | 9 +++++---- deploy/README.md | 2 +- deploy/arq/docker-compose.yml | 2 +- package.json | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/session.server.ts b/app/session.server.ts index 2c51be86..ba5506f9 100644 --- a/app/session.server.ts +++ b/app/session.server.ts @@ -1,4 +1,4 @@ -import { readFileSync } from 'node:fs' +import { readFileSync } from "node:fs"; import { createCookieSessionStorage } from "@remix-run/node"; import { createThemeSessionResolver } from "remix-themes"; @@ -12,9 +12,10 @@ export const sessionStorage = createCookieSessionStorage({ sameSite: "lax", maxAge: 604_800, // one week secrets: [ - process.env.SESSION_SECRET || - (process.env.SESSION_SECRET_FILE && readFileSync(process.env.SESSION_SECRET_FILE).toString()) || - "somebadsecret" + process.env.SESSION_SECRET || + (process.env.SESSION_SECRET_FILE && + readFileSync(process.env.SESSION_SECRET_FILE).toString()) || + "somebadsecret", ], secure: process.env.NODE_ENV === "production", }, diff --git a/deploy/README.md b/deploy/README.md index 5481d45d..755a6802 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -5,7 +5,7 @@ ## Methods -* [Haddock3 webapp with single worker and docker compose](arq/README.md) +- [Haddock3 webapp with single worker and docker compose](arq/README.md) ## Versions diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 88340c85..83934c37 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -8,7 +8,7 @@ services: dockerfile: deploy/Dockerfile restart: always entrypoint: /usr/bin/sh - command: ['-c', '/wait && npx prisma migrate deploy && npm run start'] + command: ["-c", "/wait && npx prisma migrate deploy && npm run start"] depends_on: bartender: condition: service_started diff --git a/package.json b/package.json index e4648d76..1bdaa797 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "js-yaml": "^4.1.0", "jszip": "^3.10.1", "lucide-react": "^0.321.0", - "openapi-fetch": "^0.8.2", + "openapi-fetch": "^0.8.2", "plotly.js": "^2.26.1", "react": "^18.2.0", "react-dom": "^18.2.0", From 55b9df460f3300dcec8726b9852c6080cbc54e7f Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 14:46:30 +0100 Subject: [PATCH 10/20] Untested remote slurm --- deploy/README.md | 1 + deploy/remoteslurm/README.md | 11 ++ deploy/remoteslurm/bartender-config.yaml | 134 +++++++++++++++++++++++ deploy/remoteslurm/docker-compose.yml | 114 +++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 deploy/remoteslurm/README.md create mode 100644 deploy/remoteslurm/bartender-config.yaml create mode 100644 deploy/remoteslurm/docker-compose.yml diff --git a/deploy/README.md b/deploy/README.md index 755a6802..99bb1ed7 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -6,6 +6,7 @@ ## Methods - [Haddock3 webapp with single worker and docker compose](arq/README.md) +- [Haddock3 webapp with remote slurm cluster and docker compose](remoteslurm/README.md) ## Versions diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md new file mode 100644 index 00000000..31437107 --- /dev/null +++ b/deploy/remoteslurm/README.md @@ -0,0 +1,11 @@ +# Haddock3 webapp with remote slurm cluster + +You must change the destination in the `bartender-config.yaml` to your remote Slurm cluster. + +```shell +# Must be in root of repo +cd ../.. +# Need cns executable in deploy directory, so it can be copied into the Docker image +cp deploy/cns +docker compose -f deploy/remoteslurm/docker-compose.yml up --build +``` diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml new file mode 100644 index 00000000..1107c9e6 --- /dev/null +++ b/deploy/remoteslurm/bartender-config.yaml @@ -0,0 +1,134 @@ +# This file is the configuration file for the bartender service. +# This configuration works for the webapp and the docker compose deployment. +destinations: + remoteslurm: + scheduler: + type: slurm + partition: + ssh_config: + hostname: + username: + password: + filesystem: + type: sftp + ssh_config: + hostname: + username: + password: + entry: +job_root_dir: /jobs +destination_picker: bartender.picker:pick_first +applications: + haddock3: + command_template: haddock3 workflow.cfg + upload_needs: + - workflow.cfg + runimport: + summary: Import a HADDOCK3 run. + description: > + Upload an archive of haddock3 output. + The archive should have run dir as root. + The run should have haddock3-clean and haddock3-analyse executed on it. + command_template: mkdir -p output && mv * output || true +interactive_applications: + rescore: + command_template: > + haddock3-re score + --w_elec {{w_elec|q}} --w_vdw {{w_vdw|q}} --w_desolv {{w_desolv|q}} --w_bsa {{w_bsa|q}} --w_air {{w_air|q}} + {{ capri_dir|q }} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Rescore a HADDOCK run with different weights. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + capri_dir: + type: string + w_air: + type: number + w_bsa: + type: number + w_desolv: + type: number + w_elec: + type: number + w_vdw: + type: number + required: + - module_nr + - capri_dir + - w_elec + - w_vdw + - w_desolv + - w_bsa + - w_air + type: object + reclustrmsd: + command_template: > + haddock3-re clustrmsd + {% if criterion == 'maxclust' -%} + --n_clusters {{n_clusters|q}} + {% else -%} + --clust_cutoff {{clust_cutoff|q}} + {% endif -%} + {% if min_population -%} + --min_population {{min_population|q}} + {% endif -%} + {{clustrmsd_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with RSMD and different parameters. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustrmsd_dir: + type: string + criterion: + type: string + enum: [maxclust, distance] + clust_cutoff: + type: number + n_clusters: + type: number + min_population: + type: number + required: + - module_nr + - criterion + - clustrmsd_dir + type: object + reclustfcc: + command_template: > + haddock3-re clustfcc + --clust_cutoff {{clust_cutoff|q}} --strictness {{strictness|q}} --min_population {{min_population|q}} + {{clustfcc_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with FCC and different parameters. + job_application: haddock3 + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustfcc_dir: + type: string + clust_cutoff: + type: number + strictness: + type: number + min_population: + type: number + required: + - module_nr + - clustfcc_dir + - clust_cutoff + - strictness + - min_population + type: object diff --git a/deploy/remoteslurm/docker-compose.yml b/deploy/remoteslurm/docker-compose.yml new file mode 100644 index 00000000..9d272aa9 --- /dev/null +++ b/deploy/remoteslurm/docker-compose.yml @@ -0,0 +1,114 @@ +version: "3.9" + +services: + webapp: + image: ghcr.io/i-vresse/haddock3-webapp + build: + context: ../.. + dockerfile: deploy/Dockerfile + restart: always + entrypoint: /usr/bin/sh + command: ["-c", "/wait && npx prisma migrate deploy && npm run start"] + depends_on: + bartender: + condition: service_started + webappdb: + condition: service_healthy + ports: + - "8080:8080" + environment: + BARTENDER_API_URL: "http://bartender:8000" + DATABASE_URL: postgresql://postgres:postgres@webappdb:5432/postgres + BARTENDER_PRIVATE_KEY: /certs/private_key.pem + SESSION_SECRET_FILE: /certs/session.secret + WAIT_PATHS: /certs/private_key.pem + volumes: + - bartender-certs:/certs:ro + + webappdb: + image: postgres:15.4-bullseye + restart: always + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + volumes: + - webapp-db-data:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres + interval: 2s + timeout: 3s + retries: 40 + + bartender: + image: ghcr.io/i-vresse/bartender + build: + context: .. + dockerfile: ./Dockerfile.bartenderhaddock3 + restart: always + command: sh -c '/wait && alembic upgrade head && bartender serve' + volumes: + - type: bind + source: ./bartender-config.yaml + target: /opt/bartender/config.yaml + - bartender-certs:/certs:ro + - bartender-job-data:/jobs + depends_on: + bartenderdb: + condition: service_healthy + environment: + BARTENDER_HOST: 0.0.0.0 + BARTENDER_DB_HOST: bartenderdb + BARTENDER_DB_PORT: 5432 + BARTENDER_DB_USER: postgres + BARTENDER_DB_PASS: postgres + BARTENDER_DB_BASE: postgres + BARTENDER_PUBLIC_KEY: /certs/public_key.pem + WAIT_PATHS: /certs/private_key.pem + + bartenderdb: + image: postgres:15.4-bullseye + restart: always + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + volumes: + - bartender-db-data:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres + interval: 2s + timeout: 3s + retries: 40 + + bartenderworker: + image: ghcr.io/i-vresse/bartender + build: + context: .. + dockerfile: ./Dockerfile.bartenderhaddock3 + restart: always + command: sh -c 'bartender perform' + volumes: + - type: bind + source: ./bartender-config.yaml + target: /opt/bartender/config.yaml + - bartender-job-data:/jobs + depends_on: + redis: + condition: service_started + + certmaker: + build: + context: .. + dockerfile: ./Dockerfile.openssl + volumes: + - bartender-certs:/certs + working_dir: /certs + restart: "no" + +volumes: + # Tried to use single postgres, but prisma complains about non-empty database (P3005 error). + webapp-db-data: + bartender-db-data: + bartender-job-data: + bartender-certs: From 4088f74917d3a5cbe32444489ec70bc20d721487 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 15:03:00 +0100 Subject: [PATCH 11/20] Example private ssh key + quieter log level --- deploy/Dockerfile.bartenderhaddock3 | 2 ++ deploy/arq/docker-compose.yml | 1 + deploy/remoteslurm/README.md | 5 +++++ deploy/remoteslurm/bartender-config.yaml | 4 ++-- deploy/remoteslurm/docker-compose.yml | 23 +++++++---------------- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/deploy/Dockerfile.bartenderhaddock3 b/deploy/Dockerfile.bartenderhaddock3 index 53b2cd8a..cc7d43e9 100644 --- a/deploy/Dockerfile.bartenderhaddock3 +++ b/deploy/Dockerfile.bartenderhaddock3 @@ -73,3 +73,5 @@ RUN mkdir /jobs && chown -R $USERNAME:$USERNAME /jobs # All commands from here on will be run as the bartender user and not as root. USER ${USERNAME} + +RUN mkdir /home/bartender/.ssh && chmod 700 /home/bartender/.ssh diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 83934c37..0920a521 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -65,6 +65,7 @@ services: BARTENDER_DB_USER: postgres BARTENDER_DB_PASS: postgres BARTENDER_DB_BASE: postgres + BARTENDER_LOG_LEVEL: error BARTENDER_PUBLIC_KEY: /certs/public_key.pem WAIT_PATHS: /certs/private_key.pem diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md index 31437107..60abf0fc 100644 --- a/deploy/remoteslurm/README.md +++ b/deploy/remoteslurm/README.md @@ -9,3 +9,8 @@ cd ../.. cp deploy/cns docker compose -f deploy/remoteslurm/docker-compose.yml up --build ``` + +## SSH Private key + +By default the bartender webservice will ssh/sftp to a remote machine using a password. +To use a private key, you must mount the private key into the container see [./docker-compose.yml](./docker-compose.yml) for a commented out example. diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml index 1107c9e6..3e9c0759 100644 --- a/deploy/remoteslurm/bartender-config.yaml +++ b/deploy/remoteslurm/bartender-config.yaml @@ -8,13 +8,13 @@ destinations: ssh_config: hostname: username: - password: + password: filesystem: type: sftp ssh_config: hostname: username: - password: + password: entry: job_root_dir: /jobs destination_picker: bartender.picker:pick_first diff --git a/deploy/remoteslurm/docker-compose.yml b/deploy/remoteslurm/docker-compose.yml index 9d272aa9..611d000d 100644 --- a/deploy/remoteslurm/docker-compose.yml +++ b/deploy/remoteslurm/docker-compose.yml @@ -53,6 +53,12 @@ services: target: /opt/bartender/config.yaml - bartender-certs:/certs:ro - bartender-job-data:/jobs + # To use a local private ssh key to login to the remote slurm machine + # uncomment the following lines + # The key should not have a passphrase. + # - type: bind + # source: ~/.ssh/id_rsa + # target: /home/bartender/.ssh/id_rsa depends_on: bartenderdb: condition: service_healthy @@ -63,6 +69,7 @@ services: BARTENDER_DB_USER: postgres BARTENDER_DB_PASS: postgres BARTENDER_DB_BASE: postgres + BARTENDER_LOG_LEVEL: error BARTENDER_PUBLIC_KEY: /certs/public_key.pem WAIT_PATHS: /certs/private_key.pem @@ -81,22 +88,6 @@ services: timeout: 3s retries: 40 - bartenderworker: - image: ghcr.io/i-vresse/bartender - build: - context: .. - dockerfile: ./Dockerfile.bartenderhaddock3 - restart: always - command: sh -c 'bartender perform' - volumes: - - type: bind - source: ./bartender-config.yaml - target: /opt/bartender/config.yaml - - bartender-job-data:/jobs - depends_on: - redis: - condition: service_started - certmaker: build: context: .. From 02a8f9be14d4b22a325f62982166b5126ae27e5f Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 16:06:28 +0100 Subject: [PATCH 12/20] Add image name to certmaker service --- deploy/arq/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index 0920a521..d96164eb 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -105,6 +105,7 @@ services: condition: service_started certmaker: + image: ghcr.io/i-vresse/certmaker build: context: .. dockerfile: ./Dockerfile.openssl From 7c421f694591b787a16029f543ae409211000439 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Mon, 26 Feb 2024 16:34:10 +0100 Subject: [PATCH 13/20] Making image with slurm, ssh and haddock3 --- deploy/remoteslurm/Dockerfile | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 deploy/remoteslurm/Dockerfile diff --git a/deploy/remoteslurm/Dockerfile b/deploy/remoteslurm/Dockerfile new file mode 100644 index 00000000..42634c79 --- /dev/null +++ b/deploy/remoteslurm/Dockerfile @@ -0,0 +1,58 @@ +# Dockerfile for slurm scheduler with sshd, haddock3, lightdock, gdock executables. +# Used for running as remote slurm machine against the docker-compose.yml + +# Build with: +# cd deploy +# docker build --tag ghcr.io/i-vresse/slurm -f remoteslurm/Dockerfile . +# Run with: +# docker run --publish 10022:22 ghcr.io/i-vresse/slurm + +FROM xenonmiddleware/slurm:20 + +ARG HADDOCK3_VERSION=main +ARG GDOCK_VERSION=master +ARG LIGHTDOCK_VERSION=0.9.4 + +USER root + +# Common dependencies ================================================================================================= + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential git ca-certificates curl libgfortran3 \ + wget \ + && \ + apt-get clean && rm -rf /var/lib/apt/lists/* \ + && \ + pip install --upgrade pip + +# TODO install python 3.9 from mamba or src + +# haddock3 ============================================================================================================ +WORKDIR /opt/haddock3 +RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git checkout ${HADDOCK3_VERSION} +WORKDIR /opt/haddock3/src/fcc/src +RUN make +WORKDIR /opt/haddock3 +RUN pip install -r requirements.txt && python setup.py develop +COPY ./cns /opt/haddock3/bin/cns +WORKDIR / +ENV PYTHONPATH=/opt/haddock3/src + +# lightdock ========================================================================================================== + +RUN pip install lightdock==${LIGHTDOCK_VERSION} + +# gdock ============================================================================================================= + +WORKDIR /opt/gdock +RUN git clone https://github.com/gdocking/gdock.git . && git checkout ${GDOCK_VERSION} +RUN python setup.py develop +RUN bash install.sh /opt/gdock +ENV GDOCK_PATH=/opt/gdock + +# All commands from here on will be run as the xenon user and not as root. +USER xenon + +RUN mkdir /home/xenon/.ssh && chmod 700 /home/xenon/.ssh From e1df79a13eb4d0ac38017930340fdb36027e1944 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Thu, 29 Feb 2024 16:57:05 +0100 Subject: [PATCH 14/20] Use xenon slurm 23 as from --- deploy/remoteslurm/Dockerfile | 20 +++++++++------ deploy/remoteslurm/README.md | 31 ++++++++++++++++++++++++ deploy/remoteslurm/bartender-config.yaml | 4 +-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/deploy/remoteslurm/Dockerfile b/deploy/remoteslurm/Dockerfile index 42634c79..6c216f57 100644 --- a/deploy/remoteslurm/Dockerfile +++ b/deploy/remoteslurm/Dockerfile @@ -7,7 +7,7 @@ # Run with: # docker run --publish 10022:22 ghcr.io/i-vresse/slurm -FROM xenonmiddleware/slurm:20 +FROM ghcr.io/xenon-middleware/slurm:23 ARG HADDOCK3_VERSION=main ARG GDOCK_VERSION=master @@ -20,14 +20,20 @@ USER root ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends \ - build-essential git ca-certificates curl libgfortran3 \ - wget \ + build-essential git ca-certificates curl libgfortran5 \ + wget software-properties-common \ && \ apt-get clean && rm -rf /var/lib/apt/lists/* \ && \ pip install --upgrade pip -# TODO install python 3.9 from mamba or src +# python 3.9 + /usr/local/bin/pip ========================================================================================================== + +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y python3.9 python3.9-dev python3.9-distutils && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3.9 get-pip.py && \ + rm get-pip.py # haddock3 ============================================================================================================ WORKDIR /opt/haddock3 @@ -35,20 +41,20 @@ RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git ch WORKDIR /opt/haddock3/src/fcc/src RUN make WORKDIR /opt/haddock3 -RUN pip install -r requirements.txt && python setup.py develop +RUN /usr/local/bin/pip install -r requirements.txt && python setup.py develop COPY ./cns /opt/haddock3/bin/cns WORKDIR / ENV PYTHONPATH=/opt/haddock3/src # lightdock ========================================================================================================== -RUN pip install lightdock==${LIGHTDOCK_VERSION} +RUN /usr/local/bin/pip install lightdock==${LIGHTDOCK_VERSION} # gdock ============================================================================================================= WORKDIR /opt/gdock RUN git clone https://github.com/gdocking/gdock.git . && git checkout ${GDOCK_VERSION} -RUN python setup.py develop +RUN python3.9 setup.py develop RUN bash install.sh /opt/gdock ENV GDOCK_PATH=/opt/gdock diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md index 60abf0fc..a1a5244b 100644 --- a/deploy/remoteslurm/README.md +++ b/deploy/remoteslurm/README.md @@ -14,3 +14,34 @@ docker compose -f deploy/remoteslurm/docker-compose.yml up --build By default the bartender webservice will ssh/sftp to a remote machine using a password. To use a private key, you must mount the private key into the container see [./docker-compose.yml](./docker-compose.yml) for a commented out example. + +## To test + +To test against a local machine + +Spinup a [Slurm cluster in a Docker container](https://github.com/xenon-middleware/xenon-docker-images/blob/master/slurm-23/README.md) with + +```shell +docker build -t slurmwithhaddock3 -f deploy/remoteslurm/Dockerfile deploy +docker run -p 10022:22 slurmwithhaddock3 +``` + +And in the `bartender-config.yaml` change the `remoteslurm` block to + +```yaml + scheduler: + type: slurm + ssh_config: + hostname: localhost + port: 10022 + username: xenon + password: javagat + filesystem: + type: sftp + ssh_config: + hostname: localhost + port: 10022 + username: xenon + password: javagat + entry: /home/xenon +``` diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml index 3e9c0759..925aab1c 100644 --- a/deploy/remoteslurm/bartender-config.yaml +++ b/deploy/remoteslurm/bartender-config.yaml @@ -8,13 +8,13 @@ destinations: ssh_config: hostname: username: - password: + password: filesystem: type: sftp ssh_config: hostname: username: - password: + password: entry: job_root_dir: /jobs destination_picker: bartender.picker:pick_first From d2eb4e878078e8e4cae1b5616d0e66da9d472d40 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Fri, 1 Mar 2024 08:58:58 +0100 Subject: [PATCH 15/20] Format --- deploy/remoteslurm/README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md index a1a5244b..169c146c 100644 --- a/deploy/remoteslurm/README.md +++ b/deploy/remoteslurm/README.md @@ -26,22 +26,22 @@ docker build -t slurmwithhaddock3 -f deploy/remoteslurm/Dockerfile deploy docker run -p 10022:22 slurmwithhaddock3 ``` -And in the `bartender-config.yaml` change the `remoteslurm` block to +And in the `bartender-config.yaml` change the `remoteslurm` block to ```yaml - scheduler: - type: slurm - ssh_config: - hostname: localhost - port: 10022 - username: xenon - password: javagat - filesystem: - type: sftp - ssh_config: - hostname: localhost - port: 10022 - username: xenon - password: javagat - entry: /home/xenon +scheduler: + type: slurm + ssh_config: + hostname: localhost + port: 10022 + username: xenon + password: javagat +filesystem: + type: sftp + ssh_config: + hostname: localhost + port: 10022 + username: xenon + password: javagat + entry: /home/xenon ``` From 173e6a1fcf0dbfdac6874a3cd3a55da61023bb14 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Fri, 1 Mar 2024 11:59:06 +0100 Subject: [PATCH 16/20] Start on container slurm deployment Stuck on Upload workflow gives Traceback (most recent call last): File "/usr/bin/haddock3", line 33, in sys.exit(load_entry_point('haddock3', 'console_scripts', 'haddock3')()) File "/usr/bin/haddock3", line 22, in importlib_load_entry_point for entry_point in distribution(dist_name).entry_points File "/usr/lib/python3.9/importlib/metadata.py", line 542, in distribution return Distribution.from_name(distribution_name) File "/usr/lib/python3.9/importlib/metadata.py", line 196, in from_name raise PackageNotFoundError(name) importlib.metadata.PackageNotFoundError: haddock3 Upload import also fails and has zip contents duplicated in . and output --- deploy/README.md | 1 + deploy/arq/bartender-config.yaml | 3 - .../Dockerfile | 27 +--- deploy/containerslurm/README.md | 13 ++ deploy/containerslurm/bartender-config.yaml | 130 ++++++++++++++++++ deploy/containerslurm/docker-compose.yml | 16 +++ deploy/remoteslurm/README.md | 37 +---- deploy/remoteslurm/bartender-config.yaml | 5 +- 8 files changed, 171 insertions(+), 61 deletions(-) rename deploy/{remoteslurm => containerslurm}/Dockerfile (69%) create mode 100644 deploy/containerslurm/README.md create mode 100644 deploy/containerslurm/bartender-config.yaml create mode 100644 deploy/containerslurm/docker-compose.yml diff --git a/deploy/README.md b/deploy/README.md index 99bb1ed7..56af8504 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,6 +7,7 @@ - [Haddock3 webapp with single worker and docker compose](arq/README.md) - [Haddock3 webapp with remote slurm cluster and docker compose](remoteslurm/README.md) +- [Haddock3 webapp with containerized slurm cluster and docker compose](containerslurm/README.md) ## Versions diff --git a/deploy/arq/bartender-config.yaml b/deploy/arq/bartender-config.yaml index 53a703bf..ca406621 100644 --- a/deploy/arq/bartender-config.yaml +++ b/deploy/arq/bartender-config.yaml @@ -32,7 +32,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Rescore a HADDOCK run with different weights. - job_application: haddock3 input_schema: additionalProperties: false properties: @@ -74,7 +73,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Recluster a HADDOCK run with RSMD and different parameters. - job_application: haddock3 input_schema: additionalProperties: false properties: @@ -104,7 +102,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Recluster a HADDOCK run with FCC and different parameters. - job_application: haddock3 input_schema: additionalProperties: false properties: diff --git a/deploy/remoteslurm/Dockerfile b/deploy/containerslurm/Dockerfile similarity index 69% rename from deploy/remoteslurm/Dockerfile rename to deploy/containerslurm/Dockerfile index 6c216f57..3b0641e1 100644 --- a/deploy/remoteslurm/Dockerfile +++ b/deploy/containerslurm/Dockerfile @@ -1,12 +1,4 @@ -# Dockerfile for slurm scheduler with sshd, haddock3, lightdock, gdock executables. -# Used for running as remote slurm machine against the docker-compose.yml - -# Build with: -# cd deploy -# docker build --tag ghcr.io/i-vresse/slurm -f remoteslurm/Dockerfile . -# Run with: -# docker run --publish 10022:22 ghcr.io/i-vresse/slurm - +# Dockerfile with slurm scheduler, sshd, haddock3, lightdock, gdock executables. FROM ghcr.io/xenon-middleware/slurm:23 ARG HADDOCK3_VERSION=main @@ -21,11 +13,9 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential git ca-certificates curl libgfortran5 \ - wget software-properties-common \ + wget software-properties-common gpg-agent \ && \ - apt-get clean && rm -rf /var/lib/apt/lists/* \ - && \ - pip install --upgrade pip + apt-get clean && rm -rf /var/lib/apt/lists/* # python 3.9 + /usr/local/bin/pip ========================================================================================================== @@ -41,10 +31,10 @@ RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git ch WORKDIR /opt/haddock3/src/fcc/src RUN make WORKDIR /opt/haddock3 -RUN /usr/local/bin/pip install -r requirements.txt && python setup.py develop -COPY ./cns /opt/haddock3/bin/cns +RUN /usr/local/bin/pip install -r requirements.txt && python3.9 setup.py develop +COPY cns /opt/haddock3/bin/cns WORKDIR / -ENV PYTHONPATH=/opt/haddock3/src +RUN echo 'export PYTHONPATH=/opt/haddock3/src' >> /etc/bash.bashrc # lightdock ========================================================================================================== @@ -57,8 +47,3 @@ RUN git clone https://github.com/gdocking/gdock.git . && git checkout ${GDOCK_VE RUN python3.9 setup.py develop RUN bash install.sh /opt/gdock ENV GDOCK_PATH=/opt/gdock - -# All commands from here on will be run as the xenon user and not as root. -USER xenon - -RUN mkdir /home/xenon/.ssh && chmod 700 /home/xenon/.ssh diff --git a/deploy/containerslurm/README.md b/deploy/containerslurm/README.md new file mode 100644 index 00000000..260b0361 --- /dev/null +++ b/deploy/containerslurm/README.md @@ -0,0 +1,13 @@ +## Haddock3 web app with containerized Slurm cluster + +To test against a local container with [Slurm](https://github.com/xenon-middleware/xenon-docker-images/blob/master/slurm-23/README.md) and haddock3. + +```shell +# Must be in root of repo +cd ../.. +# Need cns executable in deploy directory, so it can be copied into the Docker image +cp deploy/cns +docker compose -f deploy/remoteslurm/docker-compose.yml -f deploy/containerslurm/docker-compose.yml up --build +``` + +The haddock3 webapp should be running on http://localhost:8080 diff --git a/deploy/containerslurm/bartender-config.yaml b/deploy/containerslurm/bartender-config.yaml new file mode 100644 index 00000000..399379b8 --- /dev/null +++ b/deploy/containerslurm/bartender-config.yaml @@ -0,0 +1,130 @@ +# This file is the configuration file for the bartender service. +# This configuration works for the webapp and the docker compose deployment. +destinations: + slurm: + scheduler: + type: slurm + ssh_config: + hostname: slurm + username: xenon + password: javagat + filesystem: + type: sftp + ssh_config: + hostname: slurm + username: xenon + password: javagat + entry: /home/xenon +job_root_dir: /jobs +destination_picker: bartender.picker:pick_first +applications: + haddock3: + command_template: haddock3 workflow.cfg + upload_needs: + - workflow.cfg + runimport: + summary: Import a HADDOCK3 run. + description: > + Upload an archive of haddock3 output. + The archive should have run dir as root. + The run should have haddock3-clean and haddock3-analyse executed on it. + command_template: mkdir -p output && mv * output || true +interactive_applications: + rescore: + command_template: > + haddock3-re score + --w_elec {{w_elec|q}} --w_vdw {{w_vdw|q}} --w_desolv {{w_desolv|q}} --w_bsa {{w_bsa|q}} --w_air {{w_air|q}} + {{ capri_dir|q }} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Rescore a HADDOCK run with different weights. + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + capri_dir: + type: string + w_air: + type: number + w_bsa: + type: number + w_desolv: + type: number + w_elec: + type: number + w_vdw: + type: number + required: + - module_nr + - capri_dir + - w_elec + - w_vdw + - w_desolv + - w_bsa + - w_air + type: object + reclustrmsd: + command_template: > + haddock3-re clustrmsd + {% if criterion == 'maxclust' -%} + --n_clusters {{n_clusters|q}} + {% else -%} + --clust_cutoff {{clust_cutoff|q}} + {% endif -%} + {% if min_population -%} + --min_population {{min_population|q}} + {% endif -%} + {{clustrmsd_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with RSMD and different parameters. + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustrmsd_dir: + type: string + criterion: + type: string + enum: [maxclust, distance] + clust_cutoff: + type: number + n_clusters: + type: number + min_population: + type: number + required: + - module_nr + - criterion + - clustrmsd_dir + type: object + reclustfcc: + command_template: > + haddock3-re clustfcc + --clust_cutoff {{clust_cutoff|q}} --strictness {{strictness|q}} --min_population {{min_population|q}} + {{clustfcc_dir|q}} + && + haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ + description: Recluster a HADDOCK run with FCC and different parameters. + input_schema: + additionalProperties: false + properties: + module_nr: + type: number + clustfcc_dir: + type: string + clust_cutoff: + type: number + strictness: + type: number + min_population: + type: number + required: + - module_nr + - clustfcc_dir + - clust_cutoff + - strictness + - min_population + type: object diff --git a/deploy/containerslurm/docker-compose.yml b/deploy/containerslurm/docker-compose.yml new file mode 100644 index 00000000..6f1b8da8 --- /dev/null +++ b/deploy/containerslurm/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.9" + +services: + bartender: + volumes: + - type: bind + source: ../containerslurm/bartender-config.yaml + target: /opt/bartender/config.yaml + depends_on: + - slurm + slurm: + build: + context: .. + dockerfile: containerslurm/Dockerfile + privileged: true + \ No newline at end of file diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md index 169c146c..3b2750a7 100644 --- a/deploy/remoteslurm/README.md +++ b/deploy/remoteslurm/README.md @@ -1,4 +1,4 @@ -# Haddock3 webapp with remote slurm cluster +# Haddock3 webapp with remote Slurm cluster You must change the destination in the `bartender-config.yaml` to your remote Slurm cluster. @@ -7,41 +7,12 @@ You must change the destination in the `bartender-config.yaml` to your remote Sl cd ../.. # Need cns executable in deploy directory, so it can be copied into the Docker image cp deploy/cns -docker compose -f deploy/remoteslurm/docker-compose.yml up --build +docker compose -f deploy/slurm/docker-compose.yml up --build ``` +The haddock3 webapp should be running on http://localhost:8080 + ## SSH Private key By default the bartender webservice will ssh/sftp to a remote machine using a password. To use a private key, you must mount the private key into the container see [./docker-compose.yml](./docker-compose.yml) for a commented out example. - -## To test - -To test against a local machine - -Spinup a [Slurm cluster in a Docker container](https://github.com/xenon-middleware/xenon-docker-images/blob/master/slurm-23/README.md) with - -```shell -docker build -t slurmwithhaddock3 -f deploy/remoteslurm/Dockerfile deploy -docker run -p 10022:22 slurmwithhaddock3 -``` - -And in the `bartender-config.yaml` change the `remoteslurm` block to - -```yaml -scheduler: - type: slurm - ssh_config: - hostname: localhost - port: 10022 - username: xenon - password: javagat -filesystem: - type: sftp - ssh_config: - hostname: localhost - port: 10022 - username: xenon - password: javagat - entry: /home/xenon -``` diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml index 925aab1c..8c405a9e 100644 --- a/deploy/remoteslurm/bartender-config.yaml +++ b/deploy/remoteslurm/bartender-config.yaml @@ -1,7 +1,7 @@ # This file is the configuration file for the bartender service. # This configuration works for the webapp and the docker compose deployment. destinations: - remoteslurm: + slurm: scheduler: type: slurm partition: @@ -39,7 +39,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Rescore a HADDOCK run with different weights. - job_application: haddock3 input_schema: additionalProperties: false properties: @@ -81,7 +80,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Recluster a HADDOCK run with RSMD and different parameters. - job_application: haddock3 input_schema: additionalProperties: false properties: @@ -111,7 +109,6 @@ interactive_applications: && haddock3-analyse --is_cleaned True --inter True -m {{ module_nr|q }} -r output/ description: Recluster a HADDOCK run with FCC and different parameters. - job_application: haddock3 input_schema: additionalProperties: false properties: From 7de09cb338a1127af4699eb8a301a634e5418d25 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Sat, 2 Mar 2024 11:16:01 +0100 Subject: [PATCH 17/20] Make xenon user have working haddock3 command. By doing non-editable install + put cns in weird place --- deploy/containerslurm/Dockerfile | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/deploy/containerslurm/Dockerfile b/deploy/containerslurm/Dockerfile index 3b0641e1..f53d0ab2 100644 --- a/deploy/containerslurm/Dockerfile +++ b/deploy/containerslurm/Dockerfile @@ -23,18 +23,8 @@ RUN add-apt-repository -y ppa:deadsnakes/ppa && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y python3.9 python3.9-dev python3.9-distutils && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3.9 get-pip.py && \ - rm get-pip.py - -# haddock3 ============================================================================================================ -WORKDIR /opt/haddock3 -RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git checkout ${HADDOCK3_VERSION} -WORKDIR /opt/haddock3/src/fcc/src -RUN make -WORKDIR /opt/haddock3 -RUN /usr/local/bin/pip install -r requirements.txt && python3.9 setup.py develop -COPY cns /opt/haddock3/bin/cns -WORKDIR / -RUN echo 'export PYTHONPATH=/opt/haddock3/src' >> /etc/bash.bashrc + rm get-pip.py && \ + pip install -U pip==23.3.2 # lightdock ========================================================================================================== @@ -47,3 +37,14 @@ RUN git clone https://github.com/gdocking/gdock.git . && git checkout ${GDOCK_VE RUN python3.9 setup.py develop RUN bash install.sh /opt/gdock ENV GDOCK_PATH=/opt/gdock + +# haddock3 ============================================================================================================ +WORKDIR /opt/haddock3 +RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git checkout ${HADDOCK3_VERSION} +WORKDIR /opt/haddock3/src/fcc/src +RUN make +WORKDIR /opt/haddock3 +RUN /usr/local/bin/pip install -r requirements.txt && /usr/local/bin/pip install . && \ + mkdir -p /usr/local/lib/python3.9/bin +COPY cns /usr/local/lib/python3.9/bin/cns +WORKDIR / From be328c872fbbbba591f0626c97448a2386ff9485 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Sat, 2 Mar 2024 11:34:36 +0100 Subject: [PATCH 18/20] Scheduler runimport app locally using custom destination picker Should work after https://github.com/i-VRESSE/bartender/pull/90 is merged --- deploy/containerslurm/bartender-config.yaml | 7 ++++++- deploy/remoteslurm/bartender-config.yaml | 7 ++++++- deploy/remoteslurm/docker-compose.yml | 3 +++ deploy/remoteslurm/mypicker.py | 4 ++++ 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 deploy/remoteslurm/mypicker.py diff --git a/deploy/containerslurm/bartender-config.yaml b/deploy/containerslurm/bartender-config.yaml index 399379b8..c2c19c06 100644 --- a/deploy/containerslurm/bartender-config.yaml +++ b/deploy/containerslurm/bartender-config.yaml @@ -15,8 +15,13 @@ destinations: username: xenon password: javagat entry: /home/xenon + local: + scheduler: + type: memory + filesystem: + type: local job_root_dir: /jobs -destination_picker: bartender.picker:pick_first +destination_picker: /opt/bartender/mypicker.py:mypicker applications: haddock3: command_template: haddock3 workflow.cfg diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml index 8c405a9e..a6df8876 100644 --- a/deploy/remoteslurm/bartender-config.yaml +++ b/deploy/remoteslurm/bartender-config.yaml @@ -16,8 +16,13 @@ destinations: username: password: entry: + local: + scheduler: + type: memory + filesystem: + type: local job_root_dir: /jobs -destination_picker: bartender.picker:pick_first +destination_picker: /opt/bartender/mypicker.py:mypicker applications: haddock3: command_template: haddock3 workflow.cfg diff --git a/deploy/remoteslurm/docker-compose.yml b/deploy/remoteslurm/docker-compose.yml index 611d000d..fcc658cd 100644 --- a/deploy/remoteslurm/docker-compose.yml +++ b/deploy/remoteslurm/docker-compose.yml @@ -51,6 +51,9 @@ services: - type: bind source: ./bartender-config.yaml target: /opt/bartender/config.yaml + - type: bind + source: ./mypicker.py + target: /opt/bartender/mypicker.py - bartender-certs:/certs:ro - bartender-job-data:/jobs # To use a local private ssh key to login to the remote slurm machine diff --git a/deploy/remoteslurm/mypicker.py b/deploy/remoteslurm/mypicker.py new file mode 100644 index 00000000..3ed2b797 --- /dev/null +++ b/deploy/remoteslurm/mypicker.py @@ -0,0 +1,4 @@ +def mypicker(job_dir, application_name, submitter, context): + if application_name == 'runimport': + return 'local' + return 'slurm' From 4600e8c2397a68b882906b78e553fe4e7eca6945 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 4 Mar 2024 12:07:30 +0100 Subject: [PATCH 19/20] Container slurm deployment is able to run haddock3 and upload a run. --- deploy/README.md | 4 +++- deploy/arq/docker-compose.yml | 1 + deploy/containerslurm/Dockerfile | 9 +++++---- deploy/containerslurm/README.md | 2 +- deploy/containerslurm/bartender-config.yaml | 2 +- deploy/remoteslurm/README.md | 2 +- deploy/remoteslurm/bartender-config.yaml | 2 +- deploy/remoteslurm/docker-compose.yml | 1 + 8 files changed, 14 insertions(+), 9 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 56af8504..9ebc769e 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -13,7 +13,9 @@ The webapp version is what is currently checked out. -Versions of bartender, haddock3, gdock and lightdock are defined in the start of the [./Dockerfile.bartenderhaddock3](./Dockerfile.bartenderhaddock3) file. They can be overwritten during build with +Versions of bartender, haddock3, gdock and lightdock are defined in the start of the [./Dockerfile.bartenderhaddock3](./Dockerfile.bartenderhaddock3) file. +A version can be a git tag, branch or commit hash. +They can be overwritten during build with ```shell docker compose -f build --build-arg HADDOCK3_VERSION=v3.0.0-beta.5 diff --git a/deploy/arq/docker-compose.yml b/deploy/arq/docker-compose.yml index d96164eb..f0188ccc 100644 --- a/deploy/arq/docker-compose.yml +++ b/deploy/arq/docker-compose.yml @@ -65,6 +65,7 @@ services: BARTENDER_DB_USER: postgres BARTENDER_DB_PASS: postgres BARTENDER_DB_BASE: postgres + BARTENDER_WORKERS_COUNT: 4 BARTENDER_LOG_LEVEL: error BARTENDER_PUBLIC_KEY: /certs/public_key.pem WAIT_PATHS: /certs/private_key.pem diff --git a/deploy/containerslurm/Dockerfile b/deploy/containerslurm/Dockerfile index f53d0ab2..40a5dfff 100644 --- a/deploy/containerslurm/Dockerfile +++ b/deploy/containerslurm/Dockerfile @@ -41,10 +41,11 @@ ENV GDOCK_PATH=/opt/gdock # haddock3 ============================================================================================================ WORKDIR /opt/haddock3 RUN git clone --recursive https://github.com/haddocking/haddock3.git . && git checkout ${HADDOCK3_VERSION} -WORKDIR /opt/haddock3/src/fcc/src -RUN make -WORKDIR /opt/haddock3 +RUN cd src/fcc/src && make && cd .. && \ + # pretend fcc is installed + ln -s $PWD /usr/local/lib/python3.9/dist-packages/fcc && \ + touch __init__.py scripts/__init__.py RUN /usr/local/bin/pip install -r requirements.txt && /usr/local/bin/pip install . && \ - mkdir -p /usr/local/lib/python3.9/bin + mkdir -p /usr/local/lib/python3.9/bin COPY cns /usr/local/lib/python3.9/bin/cns WORKDIR / diff --git a/deploy/containerslurm/README.md b/deploy/containerslurm/README.md index 260b0361..53aa7748 100644 --- a/deploy/containerslurm/README.md +++ b/deploy/containerslurm/README.md @@ -7,7 +7,7 @@ To test against a local container with [Slurm](https://github.com/xenon-middlewa cd ../.. # Need cns executable in deploy directory, so it can be copied into the Docker image cp deploy/cns -docker compose -f deploy/remoteslurm/docker-compose.yml -f deploy/containerslurm/docker-compose.yml up --build +docker compose -f deploy/remoteslurm/docker-compose.yml -f deploy/containerslurm/docker-compose.yml up --build ``` The haddock3 webapp should be running on http://localhost:8080 diff --git a/deploy/containerslurm/bartender-config.yaml b/deploy/containerslurm/bartender-config.yaml index c2c19c06..af43a68e 100644 --- a/deploy/containerslurm/bartender-config.yaml +++ b/deploy/containerslurm/bartender-config.yaml @@ -17,7 +17,7 @@ destinations: entry: /home/xenon local: scheduler: - type: memory + type: eager filesystem: type: local job_root_dir: /jobs diff --git a/deploy/remoteslurm/README.md b/deploy/remoteslurm/README.md index 3b2750a7..bcb7b199 100644 --- a/deploy/remoteslurm/README.md +++ b/deploy/remoteslurm/README.md @@ -1,6 +1,6 @@ # Haddock3 webapp with remote Slurm cluster -You must change the destination in the `bartender-config.yaml` to your remote Slurm cluster. +First you **must** change the destination in the `bartender-config.yaml` to your remote Slurm cluster. ```shell # Must be in root of repo diff --git a/deploy/remoteslurm/bartender-config.yaml b/deploy/remoteslurm/bartender-config.yaml index a6df8876..9d1e9ee4 100644 --- a/deploy/remoteslurm/bartender-config.yaml +++ b/deploy/remoteslurm/bartender-config.yaml @@ -18,7 +18,7 @@ destinations: entry: local: scheduler: - type: memory + type: eager filesystem: type: local job_root_dir: /jobs diff --git a/deploy/remoteslurm/docker-compose.yml b/deploy/remoteslurm/docker-compose.yml index fcc658cd..81730b37 100644 --- a/deploy/remoteslurm/docker-compose.yml +++ b/deploy/remoteslurm/docker-compose.yml @@ -72,6 +72,7 @@ services: BARTENDER_DB_USER: postgres BARTENDER_DB_PASS: postgres BARTENDER_DB_BASE: postgres + BARTENDER_WORKERS_COUNT: 4 BARTENDER_LOG_LEVEL: error BARTENDER_PUBLIC_KEY: /certs/public_key.pem WAIT_PATHS: /certs/private_key.pem From 0adc71808e292cdcb3b049598d7ac2ff24510250 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 4 Mar 2024 12:21:14 +0100 Subject: [PATCH 20/20] Format --- deploy/containerslurm/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/containerslurm/docker-compose.yml b/deploy/containerslurm/docker-compose.yml index 6f1b8da8..806e84c1 100644 --- a/deploy/containerslurm/docker-compose.yml +++ b/deploy/containerslurm/docker-compose.yml @@ -13,4 +13,3 @@ services: context: .. dockerfile: containerslurm/Dockerfile privileged: true - \ No newline at end of file