Merge pull request #150 from lsst-dm/tickets/DM-48080

DM-48080 : Refactor Config Module
lsst-dm · Dec 19, 2024 · 4eeddde · 4eeddde
2 parents 2a59dff + 899a47e
commit 4eeddde
Show file tree

Hide file tree

Showing 45 changed files with 429 additions and 302 deletions.
diff --git a/Makefile b/Makefile
@@ -89,34 +89,34 @@ run-compose:
 	docker compose up --wait
 
 .PHONY: psql
-psql: CM_DATABASE_PORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
-psql: export CM_DATABASE_PASSWORD=INSECURE-PASSWORD
+psql: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
+psql: export DB__PASSWORD=INSECURE-PASSWORD
 psql: run-compose
-	psql postgresql://cm-service:${CM_DATABASE_PASSWORD}@localhost:${CM_DATABASE_PORT}/cm-service
+	psql postgresql://cm-service:${DB__PASSWORD}@localhost:${PGPORT}/cm-service
 
 .PHONY: test
-test: CM_DATABASE_PORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
-test: export CM_DATABASE_URL=postgresql://cm-service@localhost:${CM_DATABASE_PORT}/cm-service
-test: export CM_DATABASE_PASSWORD=INSECURE-PASSWORD
-test: export CM_DATABASE_SCHEMA=cm_service_test
+test: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
+test: export DB__URL=postgresql://cm-service@localhost:${PGPORT}/cm-service
+test: export DB__PASSWORD=INSECURE-PASSWORD
+test: export DB__SCHEMA=cm_service_test
 test: run-compose
 	python3 -m lsst.cmservice.cli.server init
 	pytest -vvv --asyncio-mode=auto --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html ${PYTEST_ARGS}
 
 .PHONY: run
-run: CM_DATABASE_PORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
-run: export CM_DATABASE_URL=postgresql://cm-service@localhost:${CM_DATABASE_PORT}/cm-service
-run: export CM_DATABASE_PASSWORD=INSECURE-PASSWORD
-run: export CM_DATABASE_ECHO=true
+run: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
+run: export DB__URL=postgresql://cm-service@localhost:${PGPORT}/cm-service
+run: export DB__PASSWORD=INSECURE-PASSWORD
+run: export DB__ECHO=true
 run: run-compose
 	python3 -m lsst.cmservice.cli.server init
 	python3 -m lsst.cmservice.cli.server run
 
 .PHONY: run-worker
-run-worker: CM_DATABASE_PORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
-run-worker: export CM_DATABASE_URL=postgresql://cm-service@localhost:${CM_DATABASE_PORT}/cm-service
-run-worker: export CM_DATABASE_PASSWORD=INSECURE-PASSWORD
-run-worker: export CM_DATABASE_ECHO=true
+run-worker: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
+run-worker: export DB__URL=postgresql://cm-service@localhost:${PGPORT}/cm-service
+run-worker: export DB__PASSWORD=INSECURE-PASSWORD
+run-worker: export DB__ECHO=true
 run-worker: run-compose
 	python3 -m lsst.cmservice.cli.server init
 	python3 -m lsst.cmservice.daemon
@@ -128,21 +128,21 @@ run-worker: run-compose
 #------------------------------------------------------------------------------
 
 .PHONY: test-sqlite
-test-sqlite: export CM_DATABASE_URL=sqlite+aiosqlite://///test_cm.db
+test-sqlite: export DB__URL=sqlite+aiosqlite://///test_cm.db
 test-sqlite:
 	python3 -m lsst.cmservice.cli.server init
 	pytest -vvv --asyncio-mode=auto --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html ${PYTEST_ARGS}
 
 .PHONY: run-sqlite
-run-sqlite: export CM_DATABASE_URL=sqlite+aiosqlite://///test_cm.db
-run-sqlite: export CM_DATABASE_ECHO=true
+run-sqlite: export DB__URL=sqlite+aiosqlite://///test_cm.db
+run-sqlite: export DB__ECHO=true
 run-sqlite:
 	python3 -m lsst.cmservice.cli.server init
 	python3 -m lsst.cmservice.cli.server run
 
 .PHONY: run-worker-sqlite
-run-worker-sqlite: export CM_DATABASE_URL=sqlite+aiosqlite://///test_cm.db
-run-worker-sqlite: export CM_DATABASE_ECHO=true
+run-worker-sqlite: export DB__URL=sqlite+aiosqlite://///test_cm.db
+run-worker-sqlite: export DB__ECHO=true
 run-worker-sqlite:
 	python3 -m lsst.cmservice.cli.server init
 	python3 -m lsst.cmservice.daemon
@@ -167,36 +167,36 @@ psql-usdf-dev: ## Connect psql client to backend Postgres (shared USDF)
 	psql
 
 .PHONY: test-usdf-dev
-test-usdf-dev: CM_DATABASE_HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')'
-test-usdf-dev: export CM_DATABASE_URL=postgresql://cm-service@${CM_DATABASE_HOST}:5432/cm-service
-test-usdf-dev: export CM_DATABASE_PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
-test-usdf-dev: export CM_DATABASE_SCHEMA=cm_service_test
+test-usdf-dev: DB__HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')'
+test-usdf-dev: export DB__URL=postgresql://cm-service@${DB__HOST}:5432/cm-service
+test-usdf-dev: export DB__PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
+test-usdf-dev: export DB__SCHEMA=cm_service_test
 test-usdf-dev:
 	pytest -vvv --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html ${PYTEST_ARGS}
 
 .PHONY: run-usdf-dev
-run-usdf-dev:CM_DATABASE_HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')'
-run-usdf-dev: export CM_DATABASE_URL=postgresql://cm-service@${CM_DATABASE_HOST}:5432/cm-service
-run-usdf-dev: export CM_DATABASE_PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
-run-usdf-dev: export CM_DATABASE_ECHO=true
+run-usdf-dev:DB__HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')'
+run-usdf-dev: export DB__URL=postgresql://cm-service@${DB__HOST}:5432/cm-service
+run-usdf-dev: export DB__PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
+run-usdf-dev: export DB__ECHO=true
 run-usdf-dev:
 	python3 -m lsst.cmservice.cli.server init
 	python3 -m lsst.cmservice.cli.server run
 
 .PHONY: run-worker-usdf-dev
-run-worker-usdf-dev: CM_DATABASE_HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')
-run-worker-usdf-dev: export CM_DATABASE_URL=postgresql://cm-service@${CM_DATABASE_HOST}:5432/cm-service
-run-worker-usdf-dev: export CM_DATABASE_PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
-run-worker-usdf-dev: export CM_DATABASE_ECHO=true
+run-worker-usdf-dev: DB__HOST=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')
+run-worker-usdf-dev: export DB__URL=postgresql://cm-service@${DB__HOST}:5432/cm-service
+run-worker-usdf-dev: export DB__PASSWORD=$(shell kubectl --cluster=usdf-cm-dev -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
+run-worker-usdf-dev: export DB__ECHO=true
 run-worker-usdf-dev:
 	python3 -m lsst.cmservice.daemon
 
-get-env-%: CM_DATABASE_HOST=$(shell kubectl --cluster=$* -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')
-get-env-%: export CM_DATABASE_URL=postgresql://cm-servicer@${CM_DATABASE_HOST}:5432/cm-service
-get-env-%: export CM_DATABASE_PASSWORD=$(shell kubectl --cluster=$* -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
-get-env-%: export CM_DATABASE_ECHO=true
+get-env-%: DB__HOST=$(shell kubectl --cluster=$* -n cm-service get svc/cm-pg-lb -o jsonpath='{..ingress[0].ip}')
+get-env-%: export DB__URL=postgresql://cm-servicer@${DB__HOST}:5432/cm-service
+get-env-%: export DB__PASSWORD=$(shell kubectl --cluster=$* -n cm-service get secret/cm-pg-app -o jsonpath='{.data.password}' | base64 --decode)
+get-env-%: export DB__ECHO=true
 get-env-%:
 	rm -f .env.$*
-	echo CM_DATABASE_URL=$${CM_DATABASE_URL} > .env.$*
-	echo CM_DATABASE_ECHO=$${CM_DATABASE_ECHO} >> .env.$*
-	echo CM_DATABASE_PASSWORD=$${CM_DATABASE_PASSWORD} >> .env.$*
+	echo DB__URL=$${DB__URL} > .env.$*
+	echo DB__ECHO=$${DB__ECHO} >> .env.$*
+	echo DB__PASSWORD=$${DB__PASSWORD} >> .env.$*
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -18,10 +18,10 @@ services:
       - lsst.cmservice.cli.server
       - init
     environment: &cmenv
-      CM_DATABASE_HOST: postgresql
-      CM_DATABASE_ECHO: true
-      CM_DATABASE_URL: postgresql://cm-service@postgresql:5432/cm-service
-      CM_DATABASE_PASSWORD: INSECURE-PASSWORD
+      DB__HOST: postgresql
+      DB__ECHO: true
+      DB__URL: postgresql://cm-service@postgresql:5432/cm-service
+      DB__PASSWORD: INSECURE-PASSWORD
     depends_on:
       postgresql:
         condition: service_healthy

diff --git a/pyproject.toml b/pyproject.toml
@@ -124,14 +124,14 @@ target-version = "py311"
 
 [tool.ruff.lint]
 ignore = [
-    "COM812",
-    "N802",
-    "N803",
-    "N806",
-    "N812",
-    "N813",
-    "N815",
-    "N816",
+    "COM812",  # missing-trailing-comma
+    "N802",  # invalid-function-name
+    "N803",  # invalid-argument-name
+    "N806",  # non-lowercase-variable-in-function
+    "N812",  # lowercase-imported-as-non-lowercase
+    "N813",  # camelcase-imported-as-constant
+    "N815",  # mixed-case-variable-in-class-scope
+    "N816",  # mixed-case-variable-in-global-scope
 ]
 select = [
     "E",  # pycodestyle

diff --git a/src/lsst/cmservice/cli/server.py b/src/lsst/cmservice/cli/server.py
@@ -20,8 +20,8 @@ def server() -> None:
 @run_with_asyncio
 async def init(*, reset: bool) -> None:  # pragma: no cover
     """Initialize the service database."""
-    logger = structlog.get_logger(config.logger_name)
-    engine = create_database_engine(config.database_url, config.database_password)
+    logger = structlog.get_logger(__name__)
+    engine = create_database_engine(config.db.url, config.db.password)
     await initialize_database(engine, logger, schema=db.Base.metadata, reset=reset)
     await engine.dispose()
 

diff --git a/src/lsst/cmservice/common/htcondor.py b/src/lsst/cmservice/common/htcondor.py
@@ -3,6 +3,7 @@
 import subprocess
 from typing import Any
 
+from ..config import config
 from .enums import StatusEnum
 from .errors import CMHTCondorCheckError, CMHTCondorSubmitError
 
@@ -49,9 +50,9 @@ def write_htcondor_script(
         should_transfer_files="Yes",
         when_to_transfer_output="ON_EXIT",
         get_env=True,
-        request_cpus=1,
-        request_memory="512M",
-        request_disk="1G",
+        request_cpus=config.htcondor.request_cpus,
+        request_memory=config.htcondor.request_mem,
+        request_disk=config.htcondor.request_disk,
     )
     options.update(**kwargs)
 
@@ -86,16 +87,16 @@ def submit_htcondor_job(
     try:
         with subprocess.Popen(
             [
-                "condor_submit",
+                config.htcondor.condor_submit_bin,
                 htcondor_script_path,
             ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-        ) as sbatch:  # pragma: no cover
-            sbatch.wait()
-            if sbatch.returncode != 0:
-                assert sbatch.stderr
-                msg = sbatch.stderr.read().decode()
+        ) as condor_submit:  # pragma: no cover
+            condor_submit.wait()
+            if condor_submit.returncode != 0:
+                assert condor_submit.stderr
+                msg = condor_submit.stderr.read().decode()
                 raise CMHTCondorSubmitError(f"Bad htcondor submit: {msg}")
     except Exception as e:
         raise CMHTCondorSubmitError(f"Bad htcondor submit: {e}") from e
@@ -126,7 +127,7 @@ def check_htcondor_job(
         if htcondor_id is None:  # pragma: no cover
             raise CMHTCondorCheckError("No htcondor_id")
         with subprocess.Popen(
-            ["condor_q", "-userlog", htcondor_id, "-af", "JobStatus", "ExitCode"],
+            [config.htcondor.condor_q_bin, "-userlog", htcondor_id, "-af", "JobStatus", "ExitCode"],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         ) as condor_q:  # pragma: no cover

diff --git a/src/lsst/cmservice/common/slurm.py b/src/lsst/cmservice/common/slurm.py
@@ -2,6 +2,7 @@
 
 import subprocess
 
+from ..config import config
 from .enums import StatusEnum
 from .errors import CMSlurmCheckError, CMSlurmSubmitError
 
@@ -62,15 +63,15 @@ def submit_slurm_job(
     try:
         with subprocess.Popen(
             [
-                "sbatch",
+                config.slurm.sbatch_bin,
                 "-o",
                 log_url,
                 "--mem",
-                "16448",
+                config.slurm.memory,
                 "--account",
-                "rubin:production",
+                config.slurm.account,
                 "-p",
-                "milano",
+                config.slurm.partition,
                 "--parsable",
                 script_url,
             ],
@@ -114,7 +115,7 @@ def check_slurm_job(
         return StatusEnum.running
     try:
         with subprocess.Popen(
-            ["sacct", "--parsable", "-b", "-j", slurm_id], stdout=subprocess.PIPE
+            [config.slurm.sacct_bin, "--parsable", "-b", "-j", slurm_id], stdout=subprocess.PIPE
         ) as sacct:  # pragma: no cover
             sacct.wait()
             if sacct.returncode != 0: