Merge pull request #8 from lsst-sqre/tickets/DM-47262

DM-47262: Add Alembic support

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.7.3
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

Dockerfile

@@ -52,6 +52,13 @@ RUN useradd --create-home appuser
 # Copy the virtualenv.
 COPY --from=install-image /opt/venv /opt/venv
 
+# Copy the Alembic configuration and migrations, and set that path as the
+# working directory so that Alembic can be run with a simple entry command
+# and no extra configuration.
+COPY --from=install-image /workdir/alembic.ini /app/alembic.ini
+COPY --from=install-image /workdir/alembic /app/alembic
+WORKDIR /app
+
 # Make sure we use the virtualenv.
 ENV PATH="/opt/venv/bin:$PATH"
 

alembic.ini

@@ -0,0 +1,15 @@
+[alembic]
+script_location = %(here)s/alembic
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+prepend_sys_path = .
+timezone = UTC
+version_path_separator = os
+
+[post_write_hooks]
+hooks = ruff ruff_format
+ruff.type = exec
+ruff.executable = ruff
+ruff.options = check --fix REVISION_SCRIPT_FILENAME
+ruff_format.type = exec
+ruff_format.executable = ruff
+ruff_format.options = format REVISION_SCRIPT_FILENAME

alembic/README.md

@@ -0,0 +1,14 @@
+# Wobbly Alembic configuration
+
+This directory contains the Alembic configuration for managing the Wobbly UWS database.
+It is installed into the Wobbly Docker image and is used to check whether the schema is up-to-date at startup of the Wobbly web service.
+It is also used by the Helm hook that updates the Wobbly UWS schema if `config.updateSchema` is enabled.
+
+## Generating new migrations
+
+For detailed instructions on how to generate a new Alembic migration, see [the Safir documentation](https://safir.lsst.io/user-guide/database/schema#create-migration).
+
+One of the files in this directory is here only to support creating migrations.
+`docker-compose.yaml` is a [docker-compose](https://docs.docker.com/compose/) configuration file that starts a PostgreSQL instance suitable for generating schema migrations.
+This file is not used at runtime.
+It is used by the tox environment described in the above documentation.

alembic/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  postgresql:
+    image: "postgres:latest"
+    hostname: "postgresql"
+    container_name: "postgresql"
+    environment:
+      POSTGRES_PASSWORD: "INSECURE"
+      POSTGRES_USER: "wobbly"
+      POSTGRES_DB: "wobbly"
+    ports:
+      - "5432:5432"

alembic/env.py

@@ -0,0 +1,22 @@
+"""Alembic migration environment."""
+
+from safir.database import run_migrations_offline, run_migrations_online
+from safir.logging import configure_alembic_logging, configure_logging
+
+from alembic import context
+from wobbly.config import config
+from wobbly.schema import SchemaBase
+
+# Configure structlog.
+configure_logging(name="wobbly", log_level=config.log_level)
+configure_alembic_logging()
+
+# Run the migrations.
+if context.is_offline_mode():
+    run_migrations_offline(SchemaBase.metadata, config.database_url)
+else:
+    run_migrations_online(
+        SchemaBase.metadata,
+        config.database_url,
+        config.database_password,
+    )

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/20241112_2307_ef0ac4e1d0cb_initial_schema.py

@@ -0,0 +1,111 @@
+"""Initial schema
+
+Revision ID: ef0ac4e1d0cb
+Revises:
+Create Date: 2024-11-12 23:07:29.449620+00:00
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "ef0ac4e1d0cb"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "job",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("service", sa.Text(), nullable=False),
+        sa.Column("owner", sa.Text(), nullable=False),
+        sa.Column(
+            "phase",
+            sa.Enum(
+                "PENDING",
+                "QUEUED",
+                "EXECUTING",
+                "COMPLETED",
+                "ERROR",
+                "UNKNOWN",
+                "HELD",
+                "SUSPENDED",
+                "ABORTED",
+                "ARCHIVED",
+                name="executionphase",
+            ),
+            nullable=False,
+        ),
+        sa.Column("run_id", sa.Text(), nullable=True),
+        sa.Column(
+            "parameters",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column("message_id", sa.Text(), nullable=True),
+        sa.Column("creation_time", sa.DateTime(), nullable=False),
+        sa.Column("start_time", sa.DateTime(), nullable=True),
+        sa.Column("end_time", sa.DateTime(), nullable=True),
+        sa.Column("destruction_time", sa.DateTime(), nullable=False),
+        sa.Column("execution_duration", sa.Integer(), nullable=True),
+        sa.Column("quote", sa.DateTime(), nullable=True),
+        sa.Column(
+            "error_type",
+            sa.Enum("TRANSIENT", "FATAL", name="errortype"),
+            nullable=True,
+        ),
+        sa.Column("error_code", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.Column("error_detail", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "by_service_owner_phase",
+        "job",
+        ["service", "owner", "phase", "creation_time"],
+        unique=False,
+    )
+    op.create_index(
+        "by_service_owner_time",
+        "job",
+        ["service", "owner", "creation_time"],
+        unique=False,
+    )
+    op.create_table(
+        "job_result",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("job_id", sa.Integer(), nullable=False),
+        sa.Column("result_id", sa.Text(), nullable=False),
+        sa.Column("sequence", sa.Integer(), nullable=False),
+        sa.Column("url", sa.Text(), nullable=False),
+        sa.Column("size", sa.Integer(), nullable=True),
+        sa.Column("mime_type", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(["job_id"], ["job.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "by_result_id", "job_result", ["job_id", "result_id"], unique=True
+    )
+    op.create_index(
+        "by_sequence", "job_result", ["job_id", "sequence"], unique=True
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("by_sequence", table_name="job_result")
+    op.drop_index("by_result_id", table_name="job_result")
+    op.drop_table("job_result")
+    op.drop_index("by_service_owner_time", table_name="job")
+    op.drop_index("by_service_owner_phase", table_name="job")
+    op.drop_table("job")
+    # ### end Alembic commands ###

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "fastapi>=0.100",
     "uvicorn[standard]",
     # Other dependencies
+    "alembic[tz]",
     "pydantic>2",
     "pydantic-settings",
     "safir[db]>=6.5.1",
@@ -104,6 +105,11 @@ python_files = ["tests/*.py", "tests/*/*.py"]
 [tool.ruff]
 extend = "ruff-shared.toml"
 
+[tool.ruff.lint.extend-per-file-ignores]
+"tests/**" = [
+    "ASYNC221", # useful to run subprocess in async tests for Alembic
+]
+
 [tool.ruff.lint.isort]
 known-first-party = ["wobbly", "tests"]
 split-on-trailing-comma = false

requirements/dev.in

@@ -21,3 +21,6 @@ pytest-cov
 
 # Documentation
 scriv
+
+# Alembic
+ruff

requirements/dev.txt

@@ -366,9 +366,9 @@ mypy-extensions==1.0.0 \
     --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
     --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
     # via mypy
-packaging==24.1 \
-    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
-    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+packaging==24.2 \
+    --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
+    --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
     # via
     #   -c requirements/main.txt
     #   pytest
@@ -494,6 +494,26 @@ requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
     # via scriv
+ruff==0.7.3 \
+    --hash=sha256:10ebce7696afe4644e8c1a23b3cf8c0f2193a310c18387c06e583ae9ef284de2 \
+    --hash=sha256:1713e2c5545863cdbfe2cbce21f69ffaf37b813bfd1fb3b90dc9a6f1963f5a8c \
+    --hash=sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344 \
+    --hash=sha256:37d0b619546103274e7f62643d14e1adcbccb242efda4e4bdb9544d7764782e9 \
+    --hash=sha256:3f36d56326b3aef8eeee150b700e519880d1aab92f471eefdef656fd57492aa2 \
+    --hash=sha256:44eb93c2499a169d49fafd07bc62ac89b1bc800b197e50ff4633aed212569299 \
+    --hash=sha256:4ba81a5f0c5478aa61674c5a2194de8b02652f17addf8dfc40c8937e6e7d79fc \
+    --hash=sha256:588a9ff2fecf01025ed065fe28809cd5a53b43505f48b69a1ac7707b1b7e4088 \
+    --hash=sha256:5d024301109a0007b78d57ab0ba190087b43dce852e552734ebf0b0b85e4fb16 \
+    --hash=sha256:5d59f0c3ee4d1a6787614e7135b72e21024875266101142a09a61439cb6e38a5 \
+    --hash=sha256:61b46049d6edc0e4317fb14b33bd693245281a3007288b68a3f5b74a22a0746d \
+    --hash=sha256:6b6224af8b5e09772c2ecb8dc9f3f344c1aa48201c7f07e7315367f6dd90ac29 \
+    --hash=sha256:6d0242ce53f3a576c35ee32d907475a8d569944c0407f91d207c8af5be5dae4e \
+    --hash=sha256:7f3eff9961b5d2644bcf1616c606e93baa2d6b349e8aa8b035f654df252c8c67 \
+    --hash=sha256:b8963cab06d130c4df2fd52c84e9f10d297826d2e8169ae0c798b6221be1d1d2 \
+    --hash=sha256:c50f95a82b94421c964fae4c27c0242890a20fe67d203d127e84fbb8013855f5 \
+    --hash=sha256:e1d1ba2e40b6e71a61b063354d04be669ab0d39c352461f3d789cac68b54a313 \
+    --hash=sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0
+    # via -r requirements/dev.in
 scriv==1.5.1 \
     --hash=sha256:30ae9ff8d144f8e0cf394c4e1d379542f1b3823767642955b54ec40dc00b32b6 \
     --hash=sha256:a3adc657733b4124fcb54527a5f3daab0d3c300de82d0fd2b9b297b243151b78

requirements/main.txt

@@ -40,7 +40,9 @@ aiokafka==0.12.0 \
 alembic==1.14.0 \
     --hash=sha256:99bd884ca390466db5e27ffccff1d179ec5c05c965cfefc0607e69f9e411cb25 \
     --hash=sha256:b00892b53b3642d0b8dbedba234dbf1924b69be83a9a769d5a624b01094e304b
-    # via safir
+    # via
+    #   wobbly (pyproject.toml)
+    #   safir
 annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
     --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
@@ -289,9 +291,9 @@ fastavro==1.9.7 \
     # via
     #   dataclasses-avroschema
     #   python-schema-registry-client
-faststream==0.5.28 \
-    --hash=sha256:620f3edd2b85ac3d329726907540027abebc939ea7f16d209ef18de7d6e82f49 \
-    --hash=sha256:67794ffa8054169488f211832c50c3a992d897ebba415032a327372e019bda4c
+faststream==0.5.29 \
+    --hash=sha256:055942ee5b0f27f93a8cb515d55917caa2de18a41591f466367ba32eef1b3565 \
+    --hash=sha256:b4f8709dc5c1b06508b83ca697478eee419a8284a4d669f2f436838d59f0b5ad
     # via safir
 gidgethub==5.3.0 \
     --hash=sha256:4dd92f2252d12756b13f9dd15cde322bfb0d625b6fb5d680da1567ec74b462c0 \
@@ -658,9 +660,9 @@ markupsafe==3.0.2 \
     --hash=sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430 \
     --hash=sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50
     # via mako
-packaging==24.1 \
-    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
-    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+packaging==24.2 \
+    --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
+    --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
     # via aiokafka
 pycparser==2.22 ; platform_python_implementation != 'PyPy' \
     --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
@@ -954,9 +956,9 @@ rpds-py==0.21.0 \
     # via
     #   jsonschema
     #   referencing
-safir==6.5.1 \
-    --hash=sha256:8c4b85f07e2343848f598bbcba832fec6725219c4234e2a05e97e5623347d6bf \
-    --hash=sha256:e00167de168115477309436f136bd3c01623dfc362b9d32bc6c87ea65857f993
+safir==7.0.0 \
+    --hash=sha256:a43143ec1a8b569a27b0731c96393b91df1f0a8a7e811efe9e59f303087dc679 \
+    --hash=sha256:b781c5d3a7db07d91be2dffcf5842406b0b40d75a130c167b1527147c754dddc
     # via wobbly (pyproject.toml)
 safir-logging==6.5.1 \
     --hash=sha256:b056306de26627e29bd6a6d04b1144456a1319ec0e15a67ebbc12b43362a27cd \