Merge pull request #366 from lsst-sqre/tickets/DM-32595

[DM-32595] Convert to async SQLAlchemy
lsst-sqre · Dec 2, 2021 · 75521a9 · 75521a9
2 parents 5759ce1 + d0a07cb
commit 75521a9
Show file tree

Hide file tree

Showing 92 changed files with 9,834 additions and 10,651 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -85,8 +85,6 @@ jobs:
         python:
           - "3.9"
           - "3.10"
-        database:
-          - PostgreSQL
 
     steps:
       - uses: actions/checkout@v2
@@ -117,13 +115,8 @@ jobs:
           restore-keys: |
             tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-
 
-      - name: Run tox (SQLite)
+      - name: Run tox
         run: tox -e py,coverage-report,typing
-        if: matrix.database == 'SQLite'
-
-      - name: Run tox (PostgreSQL)
-        run: tox -e docker,coverage-report,typing
-        if: matrix.database == 'PostgreSQL'
 
   docs:
     runs-on: ubuntu-latest

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -43,16 +43,16 @@ repos:
     hooks:
       - id: eslint
         additional_dependencies:
-          - '@babel/eslint-parser@7.16.0'
+          - '@babel/eslint-parser@7.16.3'
           - '@babel/preset-react@7.16.0'
-          - eslint@8.2.0
-          - eslint-config-airbnb@18.2.1
+          - eslint@8.3.0
+          - eslint-config-airbnb@19.0.1
           - eslint-config-prettier@8.3.0
           - eslint-config-wesbos@2.1.0
           - eslint-plugin-html@6.2.0
-          - eslint-plugin-import@2.25.2
-          - eslint-plugin-jsx-a11y@6.4.1
+          - eslint-plugin-import@2.25.3
+          - eslint-plugin-jsx-a11y@6.5.1
           - eslint-plugin-prettier@4.0.0
-          - eslint-plugin-react@7.26.1
-          - eslint-plugin-react-hooks@4.2.0
+          - eslint-plugin-react@7.27.1
+          - eslint-plugin-react-hooks@4.3.0
           - prettier@2.4.1
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,6 +2,16 @@
 Change log
 ##########
 
+3.4.0 (2021-12-02)
+==================
+
+- Gafaelfawr now uses async SQLAlchemy for all database calls, which avoids latency affecting the whole process when a request requires database queries or writes.
+- Internal and notebook tokens are now acquired, when needed, while holding a per-user cache lock.
+  This means that when a flood of requests that all require a delegated token come in at the same time, a given Gafaelfawr process allows only the first request to proceed and blocks the rest until it completes.
+  All the other requests are then served from the cache.
+  This fixes a deadlock observed in previous versions of Gafaelfawr under heavy load from a single user who does not have a cached delegated token.
+- Update dependencies.
+
 3.3.0 (2021-11-11)
 ==================
 

diff --git a/docs/api.rst b/docs/api.rst
@@ -20,6 +20,8 @@ API reference
 
 .. automodapi:: gafaelfawr.dependencies.context
 
+.. automodapi:: gafaelfawr.dependencies.db_session
+
 .. automodapi:: gafaelfawr.dependencies.redis
 
 .. automodapi:: gafaelfawr.dependencies.return_url
@@ -64,6 +66,8 @@ API reference
 
 .. automodapi:: gafaelfawr.services.token
 
+.. automodapi:: gafaelfawr.services.token_cache
+
 .. automodapi:: gafaelfawr.storage.admin
 
 .. automodapi:: gafaelfawr.storage.base
@@ -76,13 +80,9 @@ API reference
 
 .. automodapi:: gafaelfawr.storage.token
 
-.. automodapi:: gafaelfawr.storage.transaction
-
 .. automodapi:: gafaelfawr.templates
    :include-all-objects:
 
-.. automodapi:: gafaelfawr.token_cache
-
 .. automodapi:: gafaelfawr.util
 
 .. automodapi:: gafaelfawr.verify
diff --git a/requirements/dev.in b/requirements/dev.in
@@ -12,21 +12,20 @@ coverage[toml]
 diagrams
 documenteer
 holdup
-lsst-sphinx-bootstrap-theme<0.3
-mockaioredis
+lsst-sphinx-bootstrap-theme
 mypy
 pre-commit
 pytest
 pytest-asyncio
 pytest-cov
-pytest-xdist[psutil]
 respx
 selenium
 selenium-wire
 seqdiag
 sphinx-automodapi
 sphinx-click
 sphinx-prompt
+sqlalchemy[mypy]
 types-cachetools
 types-PyYAML
 

diff --git a/requirements/dev.txt b/requirements/dev.txt
diff --git a/requirements/main.in b/requirements/main.in
@@ -8,7 +8,6 @@
 # These dependencies are for fastapi including some optional features.
 aiofiles
 fastapi
-fastapi-sqlalchemy
 python-multipart
 starlette
 uvicorn[standard]
@@ -18,12 +17,12 @@ jinja2<3
 
 # Other dependencies.
 aioredis
+asyncpg
 cachetools
 click
 cryptography
 kubernetes
 httpx
-psycopg2
 pydantic
 PyJWT
 pyyaml

diff --git a/requirements/main.txt b/requirements/main.txt
@@ -4,17 +4,17 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/main.txt requirements/main.in
 #
-aiofiles==0.7.0 \
-    --hash=sha256:a1c4fc9b2ff81568c83e21392a82f344ea9d23da906e4f6a52662764545e19d4 \
-    --hash=sha256:c67a6823b5f23fcab0a2595a289cec7d8c863ffcb4322fb8cd6b90400aedfdbc
+aiofiles==0.8.0 \
+    --hash=sha256:7a973fc22b29e9962d0897805ace5856e6a566ab1f0c8e5c91ff6c866519c937 \
+    --hash=sha256:8334f23235248a3b2e83b2c3a78a22674f39969b96397126cc93664d9a901e59
     # via -r requirements/main.in
 aioredis==2.0.0 \
     --hash=sha256:3a2de4b614e6a5f8e104238924294dc4e811aefbe17ddf52c04a93cbf06e67db \
     --hash=sha256:9921d68a3df5c5cdb0d5b49ad4fc88a4cfdd60c108325df4f0066e8410c55ffb
     # via -r requirements/main.in
-anyio==3.3.4 \
-    --hash=sha256:4fd09a25ab7fa01d34512b7249e366cd10358cdafc95022c7ff8c8f8a5026d66 \
-    --hash=sha256:67da67b5b21f96b9d3d65daa6ea99f5d5282cb09f50eb4456f8fb51dffefc3ff
+anyio==3.4.0 \
+    --hash=sha256:24adc69309fb5779bc1e06158e143e0b6d2c56b302a3ac3de3083c705a6ed39d \
+    --hash=sha256:2855a9423524abcdd652d942f8932fda1735210f77a6b392eafd9ff34d3fe020
     # via
     #   httpcore
     #   starlette
@@ -26,6 +26,34 @@ async-timeout==4.0.1 \
     --hash=sha256:a22c0b311af23337eb05fcf05a8b51c3ea53729d46fb5460af62bee033cec690 \
     --hash=sha256:b930cb161a39042f9222f6efb7301399c87eeab394727ec5437924a36d6eef51
     # via aioredis
+asyncpg==0.25.0 \
+    --hash=sha256:0a61fb196ce4dae2f2fa26eb20a778db21bbee484d2e798cb3cc988de13bdd1b \
+    --hash=sha256:18d49e2d93a7139a2fdbd113e320cc47075049997268a61bfbe0dde680c55471 \
+    --hash=sha256:191fe6341385b7fdea7dbdcf47fd6db3fd198827dcc1f2b228476d13c05a03c6 \
+    --hash=sha256:1a70783f6ffa34cc7dd2de20a873181414a34fd35a4a208a1f1a7f9f695e4ec4 \
+    --hash=sha256:2633331cbc8429030b4f20f712f8d0fbba57fa8555ee9b2f45f981b81328b256 \
+    --hash=sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a \
+    --hash=sha256:4327f691b1bdb222df27841938b3e04c14068166b3a97491bec2cb982f49f03e \
+    --hash=sha256:43cde84e996a3afe75f325a68300093425c2f47d340c0fc8912765cf24a1c095 \
+    --hash=sha256:52fab7f1b2c29e187dd8781fce896249500cf055b63471ad66332e537e9b5f7e \
+    --hash=sha256:56d88d7ef4341412cd9c68efba323a4519c916979ba91b95d4c08799d2ff0c09 \
+    --hash=sha256:5e4105f57ad1e8fbc8b1e535d8fcefa6ce6c71081228f08680c6dea24384ff0e \
+    --hash=sha256:63f8e6a69733b285497c2855464a34de657f2cccd25aeaeeb5071872e9382540 \
+    --hash=sha256:649e2966d98cc48d0646d9a4e29abecd8b59d38d55c256d5c857f6b27b7407ac \
+    --hash=sha256:6f8f5fc975246eda83da8031a14004b9197f510c41511018e7b1bedde6968e92 \
+    --hash=sha256:72a1e12ea0cf7c1e02794b697e3ca967b2360eaa2ce5d4bfdd8604ec2d6b774b \
+    --hash=sha256:739bbd7f89a2b2f6bc44cb8bf967dab12c5bc714fcbe96e68d512be45ecdf962 \
+    --hash=sha256:863d36eba4a7caa853fd7d83fad5fd5306f050cc2fe6e54fbe10cdb30420e5e9 \
+    --hash=sha256:a738f1b2876f30d710d3dc1e7858160a0afe1603ba16bf5f391f5316eb0ed855 \
+    --hash=sha256:a84d30e6f850bac0876990bcd207362778e2208df0bee8be8da9f1558255e634 \
+    --hash=sha256:acb311722352152936e58a8ee3c5b8e791b24e84cd7d777c414ff05b3530ca68 \
+    --hash=sha256:beaecc52ad39614f6ca2e48c3ca15d56e24a2c15cbfdcb764a4320cc45f02fd5 \
+    --hash=sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3 \
+    --hash=sha256:bf6dc9b55b9113f39eaa2057337ce3f9ef7de99a053b8a16360395ce588925cd \
+    --hash=sha256:ddb4c3263a8d63dcde3d2c4ac1c25206bfeb31fa83bd70fd539e10f87739dee4 \
+    --hash=sha256:f55918ded7b85723a5eaeb34e86e7b9280d4474be67df853ab5a7fa0cc7c6bf2 \
+    --hash=sha256:fe471ccd915b739ca65e2e4dbd92a11b44a5b37f2e38f70827a1c147dafe0fa8
+    # via -r requirements/main.in
 cachetools==4.2.4 \
     --hash=sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693 \
     --hash=sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1
@@ -92,9 +120,9 @@ cffi==1.15.0 \
     --hash=sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997 \
     --hash=sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796
     # via cryptography
-charset-normalizer==2.0.7 \
-    --hash=sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0 \
-    --hash=sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b
+charset-normalizer==2.0.8 \
+    --hash=sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0 \
+    --hash=sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405
     # via
     #   httpx
     #   requests
@@ -116,6 +144,7 @@ cryptography==36.0.0 \
     --hash=sha256:684993ff6f67000a56454b41bdc7e015429732d65a52d06385b6e9de6181c71e \
     --hash=sha256:6fbbbb8aab4053fa018984bb0e95a16faeb051dd8cca15add2a27e267ba02b58 \
     --hash=sha256:8982c19bb90a4fa2aad3d635c6d71814e38b643649b4000a8419f8691f20ac44 \
+    --hash=sha256:9511416e85e449fe1de73f7f99b21b3aa04fba4c4d335d30c486ba3756e3a2a6 \
     --hash=sha256:97199a13b772e74cdcdb03760c32109c808aff7cd49c29e9cf4b7754bb725d1d \
     --hash=sha256:a776bae1629c8d7198396fd93ec0265f8dd2341c553dc32b976168aaf0e6a636 \
     --hash=sha256:aa94d617a4cd4cdf4af9b5af65100c036bce22280ebb15d8b5262e8273ebc6ba \
@@ -132,10 +161,6 @@ fastapi==0.70.0 \
     # via
     #   -r requirements/main.in
     #   safir
-fastapi-sqlalchemy==0.2.1 \
-    --hash=sha256:7a9d44e46cbc73c3f5ee8c444f7e0bcd3d01370a878740abd4cd4d2e900ce9af \
-    --hash=sha256:d3bfc6d9388a73a2c3726bc6bd7764cd82debfa71c16e3991c544b9701f48d96
-    # via -r requirements/main.in
 google-auth==2.3.3 \
     --hash=sha256:a348a50b027679cb7dae98043ac8dbcc1d7951f06d8387496071a1e05a2465c0 \
     --hash=sha256:d83570a664c10b97a1dc6f8df87e5fdfff012f48f62be131e449c20dfc32630e
@@ -236,9 +261,9 @@ jinja2==2.11.3 \
     --hash=sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 \
     --hash=sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6
     # via -r requirements/main.in
-kubernetes==19.15.0 \
-    --hash=sha256:08c93f300a9837104282ecc81458b903a56444c5c1ec3d990d237557312af47f \
-    --hash=sha256:52312adda60d92ba45b325f2c1505924656389222005f7e089718e1ad03bc07f
+kubernetes==20.13.0 \
+    --hash=sha256:7d32ea7f555813a141a0e912e7695d88f7213bb632a860ba79a963b43f7a6b18 \
+    --hash=sha256:ce5e881c13dc56f21a243804f90bc3c507af93c380f505c00e392c823968e4de
     # via -r requirements/main.in
 markupsafe==2.0.1 \
     --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \
@@ -315,19 +340,6 @@ oauthlib==3.1.1 \
     --hash=sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc \
     --hash=sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3
     # via requests-oauthlib
-psycopg2==2.9.2 \
-    --hash=sha256:26322c3f114de1f60c1b0febf8fdd595c221b4f624524178f515d07350a71bd1 \
-    --hash=sha256:6796ac614412ce374587147150e56d03b7845c9e031b88aacdcadc880e81bb38 \
-    --hash=sha256:77b9105ef37bc005b8ffbcb1ed6d8685bb0e8ce84773738aa56421a007ec5a7a \
-    --hash=sha256:77d09a79f9739b97099d2952bbbf18eaa4eaf825362387acbb9552ec1b3fa228 \
-    --hash=sha256:91c7fd0fe9e6c118e8ff5b665bc3445781d3615fa78e131d0b4f8c85e8ca9ec8 \
-    --hash=sha256:a761b60da0ecaf6a9866985bcde26327883ac3cdb90535ab68b8d784f02b05ef \
-    --hash=sha256:a84da9fa891848e0270e8e04dcca073bc9046441eeb47069f5c0e36783debbea \
-    --hash=sha256:b8816c6410fa08d2a022e4e38d128bae97c1855e176a00493d6ec62ccd606d57 \
-    --hash=sha256:dfc32db6ce9ecc35a131320888b547199f79822b028934bb5b332f4169393e15 \
-    --hash=sha256:f65cba7924363e0d2f416041b48ff69d559548f2cb168ff972c54e09e1e64db8 \
-    --hash=sha256:fd7ddab7d6afee4e21c03c648c8b667b197104713e57ec404d5b74097af21e31
-    # via -r requirements/main.in
 pyasn1==0.4.8 \
     --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
     --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba
@@ -436,9 +448,9 @@ rfc3986[idna2008]==1.5.0 \
     --hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \
     --hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97
     # via httpx
-rsa==4.7.2 \
-    --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \
-    --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9
+rsa==4.8 \
+    --hash=sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17 \
+    --hash=sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb
     # via google-auth
 safir==2.2.0 \
     --hash=sha256:ac8493c9a556c50e0a4e4062f4547005f4fe73487c8e82cfb159a8866ccf02f1 \
@@ -496,20 +508,17 @@ sqlalchemy==1.4.27 \
     --hash=sha256:ec1c908fa721f2c5684900cc8ff75555b1a5a2ae4f5a5694eb0e37a5263cea44 \
     --hash=sha256:fa52534076394af7315306a8701b726a6521b591d95e8f4e5121c82f94790e8d \
     --hash=sha256:fd421a14edf73cfe01e8f51ed8966294ee3b3db8da921cacc88e497fd6e977af
-    # via
-    #   -r requirements/main.in
-    #   fastapi-sqlalchemy
+    # via -r requirements/main.in
 starlette==0.16.0 \
     --hash=sha256:38eb24bf705a2c317e15868e384c1b8a12ca396e5a3c3a003db7e667c43f939f \
     --hash=sha256:e1904b5d0007aee24bdd3c43994be9b3b729f4f58e740200de1d623f8c3a8870
     # via
     #   -r requirements/main.in
     #   fastapi
-    #   fastapi-sqlalchemy
     #   safir
-structlog==21.3.0 \
-    --hash=sha256:063216becff8e6f6558122a9b00734f7e50bfef309eb730c85a52c74ed861a96 \
-    --hash=sha256:4da2aec0aebf6dee7beb884eb0fda26ed9d6cce5338fcd523e8597d0f1826746
+structlog==21.4.0 \
+    --hash=sha256:305a66201f9605a2e8a2595271a446f258175901c09c01e4c2c2a8ac5b68edf1 \
+    --hash=sha256:6ed8fadb27cf8362be0e606f5e79ccdd3b1e879aac65f9dc0ac3033fd013a7be
     # via
     #   -r requirements/main.in
     #   safir
@@ -608,9 +617,9 @@ websockets==10.1 \
     # via uvicorn
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==59.2.0 \
-    --hash=sha256:157d21de9d055ab9e8ea3186d91e7f4f865e11f42deafa952d90842671fc2576 \
-    --hash=sha256:4adde3d1e1c89bde1c643c64d89cdd94cbfd8c75252ee459d4500bccb9c7d05d
+setuptools==59.4.0 \
+    --hash=sha256:b4c634615a0cf5b02cf83c7bedffc8da0ca439f00e79452699454da6fbd4153d \
+    --hash=sha256:feb5ff19b354cde9efd2344ef6d5e79880ce4be643037641b49508bbb850d060
     # via
     #   google-auth
     #   kubernetes
diff --git a/setup.cfg b/setup.cfg
@@ -49,6 +49,7 @@ ignore = E203, W503
 disallow_untyped_defs = True
 disallow_incomplete_defs = True
 ignore_missing_imports = True
+plugins = sqlalchemy.ext.mypy.plugin
 show_error_codes = True
 strict_equality = True
 warn_redundant_casts = True

diff --git a/src/gafaelfawr/config.py b/src/gafaelfawr/config.py
@@ -588,16 +588,19 @@ def from_file(cls, path: str) -> Config:
             oidc_secret = cls._load_secret(path).decode()
 
         # The database URL may have a separate secret in database_password, in
-        # which case it needs to be added to the URL.
-        database_url = settings.database_url
+        # which case it needs to be added to the URL.  It also needs to be
+        # configured to use asyncpg.
+        parsed_url = urlparse(settings.database_url)
+        if parsed_url.scheme == "postgresql":
+            parsed_url = parsed_url._replace(scheme="postgresql+asyncpg")
         if settings.database_password:
-            parsed_url = urlparse(database_url)
             database_password = settings.database_password.get_secret_value()
             database_netloc = (
                 f"{parsed_url.username}:{database_password}"
                 f"@{parsed_url.hostname}"
             )
-            database_url = parsed_url._replace(netloc=database_netloc).geturl()
+            parsed_url = parsed_url._replace(netloc=database_netloc)
+        database_url = parsed_url.geturl()
 
         # If there is an OpenID Connect server configuration, load it from a
         # file in JSON format.  (It contains secrets.)