Convert to the new Safir Kafka metrics system

Log events via the new Safir Kafka metrics system instead of using
OpenTelemetry. This uses Safir library support to create Avro schemas
for Kafka and export structured Pydantic events to Kafka.

alembic/gafaelfawr.yaml

@@ -31,3 +31,14 @@ knownScopes:
 
 # A minimal mapping sufficient for Gafaelfawr to start.
 groupMapping: {}
+
+# A minimal metrics configuration so that Gafaelfawr will start.
+metrics:
+  metricsEvents:
+    appName: "gafaelfawr"
+    disable: true
+  schemaManager:
+    registryUrl: "https://example.com/schema"
+  kafka:
+    bootstrapServers: "kafka"
+    securityProtocol: "PLAINTEXT"

changelog.d/20240814_105528_rra_DM_45518.md

@@ -1,6 +1,6 @@
 ### New features
 
-- Add support for exporting metrics to an OpenTelemetry collector. The initial set of metrics is limited to login metrics, token delegation, and counts of active sessions and user tokens.
+- Add support for exporting metrics to Kafka using the new event metrics support in [Safir](https://safir.lsst.io/). The initial set of events is limited to login metrics, authentications to services, and counts of active sessions and user tokens.
 
 ### Bug fixes
 

docs/_rst_epilog.rst

@@ -3,7 +3,6 @@
 .. _Keycloak: https://www.keycloak.org/
 .. _Kopf: https://kopf.readthedocs.io/en/stable/
 .. _mypy: https://mypy.readthedocs.io/en/stable/
-.. _OpenTelemetry: https://opentelemetry.io/
 .. _Phalanx: https://phalanx.lsst.io/
 .. _pre-commit: https://pre-commit.com
 .. _pytest: https://docs.pytest.org/en/latest/

docs/dev/internals.rst

@@ -29,6 +29,9 @@ Python internal API
 .. automodapi:: gafaelfawr.dependencies.return_url
    :include-all-objects:
 
+.. automodapi:: gafaelfawr.events
+   :include-all-objects:
+
 .. automodapi:: gafaelfawr.exceptions
    :include-all-objects:
 
@@ -38,9 +41,6 @@ Python internal API
 .. automodapi:: gafaelfawr.keypair
    :include-all-objects:
 
-.. automodapi:: gafaelfawr.metrics
-   :include-all-objects:
-
 .. automodapi:: gafaelfawr.middleware.state
    :include-all-objects:
 

docs/documenteer.toml

@@ -1,6 +1,6 @@
 [project]
 title = "Gafaelfawr"
-copyright = "2020-2022 Association of Universities for Research in Astronomy, Inc. (AURA)"
+copyright = "2020-2024 Association of Universities for Research in Astronomy, Inc. (AURA)"
 
 [project.openapi]
 openapi_path = "_static/openapi.json"
@@ -84,7 +84,6 @@ bonsai = "https://bonsai.readthedocs.io/en/latest"
 cryptography = "https://cryptography.io/en/latest"
 jwt = "https://pyjwt.readthedocs.io/en/latest"
 kopf = "https://kopf.readthedocs.io/en/stable"
-opentelemetry = "https://opentelemetry-python.readthedocs.io/en/latest"
 python = "https://docs.python.org/3"
 redis = "https://redis-py.readthedocs.io/en/stable"
 safir = "https://safir.lsst.io"

pyproject.toml

@@ -38,17 +38,16 @@ dependencies = [
     "kopf",
     "kubernetes-asyncio",
     "jinja2",
-    "opentelemetry-api",
-    "opentelemetry-exporter-otlp-proto-grpc",
-    "opentelemetry-sdk",
     "pydantic>2",
     "pydantic-settings",
     "pyjwt",
     "pyyaml",
     "redis>=4.2.0",
-    "safir[db,kubernetes]>=6.4.0",
+    "safir[db,kubernetes] @ git+https://github.com/lsst-sqre/safir@tickets/DM-46821#subdirectory=safir",
     "sqlalchemy>=2.0.0",
     "structlog",
+    # Temporary constraint to avoid faststream bug.
+    "anyio!=4.6.2.post1",
 ]
 dynamic = ["version"]
 

requirements/dev.txt

@@ -10,9 +10,9 @@ annotated-types==0.7.0 \
     # via
     #   -c requirements/main.txt
     #   pydantic
-anyio==4.6.2.post1 \
-    --hash=sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c \
-    --hash=sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d
+anyio==4.6.1 \
+    --hash=sha256:0632863c9044798a494a05cab0b159dfad6a3f064094863a45878320eb4e8ed2 \
+    --hash=sha256:936e6613a08e8f71a300cfffca1c1c0806335607247696ac45f9b32c63bfb9aa
     # via
     #   -c requirements/main.txt
     #   httpx
@@ -660,11 +660,10 @@ imagesize==1.4.1 \
     --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \
     --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a
     # via sphinx
-importlib-metadata==8.4.0 \
-    --hash=sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1 \
-    --hash=sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5
+importlib-metadata==8.5.0 \
+    --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \
+    --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7
     # via
-    #   -c requirements/main.txt
     #   jupyter-cache
     #   myst-nb
 iniconfig==2.0.0 \
@@ -700,12 +699,15 @@ jsonschema==4.23.0 \
     --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \
     --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566
     # via
+    #   -c requirements/main.txt
     #   nbformat
     #   sphinxcontrib-redoc
 jsonschema-specifications==2024.10.1 \
     --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \
     --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf
-    # via jsonschema
+    # via
+    #   -c requirements/main.txt
+    #   jsonschema
 jupyter-cache==1.0.0 \
     --hash=sha256:594b1c4e29b488b36547e12477645f489dbdc62cc939b2408df5679f79245078 \
     --hash=sha256:d0fa7d7533cd5798198d8889318269a8c1382ed3b22f622c09a9356521f48687
@@ -898,6 +900,7 @@ packaging==24.1 \
     --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
     --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
     # via
+    #   -c requirements/main.txt
     #   ipykernel
     #   pydata-sphinx-theme
     #   pytest
@@ -1345,6 +1348,7 @@ referencing==0.35.1 \
     --hash=sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c \
     --hash=sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de
     # via
+    #   -c requirements/main.txt
     #   jsonschema
     #   jsonschema-specifications
 requests==2.32.3 \
@@ -1465,6 +1469,7 @@ rpds-py==0.20.0 \
     --hash=sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a \
     --hash=sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8
     # via
+    #   -c requirements/main.txt
     #   jsonschema
     #   referencing
 scriv==1.5.1 \
@@ -1628,7 +1633,9 @@ sphinxext-rediraffe==0.2.7 \
     --hash=sha256:9e430a52d4403847f4ffb3a8dd6dfc34a9fe43525305131f52ed899743a5fd8c
     # via documenteer
 sqlalchemy==2.0.36 \
+    --hash=sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763 \
     --hash=sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436 \
+    --hash=sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2 \
     --hash=sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588 \
     --hash=sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e \
     --hash=sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959 \
@@ -1638,26 +1645,38 @@ sqlalchemy==2.0.36 \
     --hash=sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7 \
     --hash=sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971 \
     --hash=sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c \
+    --hash=sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d \
+    --hash=sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb \
     --hash=sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f \
+    --hash=sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5 \
+    --hash=sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346 \
     --hash=sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e \
     --hash=sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5 \
+    --hash=sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08 \
     --hash=sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793 \
     --hash=sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88 \
+    --hash=sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686 \
+    --hash=sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b \
     --hash=sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2 \
     --hash=sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28 \
+    --hash=sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d \
     --hash=sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5 \
     --hash=sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a \
     --hash=sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a \
+    --hash=sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3 \
+    --hash=sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf \
     --hash=sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5 \
     --hash=sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c \
     --hash=sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07 \
     --hash=sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa \
     --hash=sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06 \
     --hash=sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1 \
+    --hash=sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff \
     --hash=sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687 \
     --hash=sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb \
     --hash=sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44 \
-    --hash=sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e
+    --hash=sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e \
+    --hash=sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53
     # via
     #   -c requirements/main.txt
     #   -r requirements/dev.in
@@ -1807,9 +1826,7 @@ wsproto==1.2.0 \
 zipp==3.20.2 \
     --hash=sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350 \
     --hash=sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29
-    # via
-    #   -c requirements/main.txt
-    #   importlib-metadata
+    # via importlib-metadata
 zstandard==0.23.0 \
     --hash=sha256:034b88913ecc1b097f528e42b539453fa82c3557e414b3de9d5632c80439a473 \
     --hash=sha256:0a7f0804bb3799414af278e9ad51be25edf67f78f916e08afdb983e74161b916 \