Merge pull request #236 from ebmdatalab/benbc/cache-github-requests

Cache GitHub requests

.gitignore

@@ -6,3 +6,4 @@
 __pycache__/
 github.db
 htmlcov
+github-cache.sqlite

DEVELOPERS.md

@@ -71,7 +71,7 @@ e.g `just metrics prs` to run metrics/tasks/prs.py
 
 All tasks are defined in `metrics/tasks` and must have a `main()` function that takes no arguments.
 
-### Fast debug mode
+### Speeding up development
 
 You can set a flag to trigger a fast mode which only retrieves and handful of PRs
 but allows the main code paths to be tested quickly.
@@ -80,6 +80,23 @@ but allows the main code paths to be tested quickly.
 DEBUG_FAST=t just metrics prs
 ```
 
+Alternatively you can turn on caching of GitHub API requests.
+This is particularly useful when iterating on metric definitions
+without changing the data that we retrieve from the API.
+
+```
+DEBUG_CACHE=t just metrics prs
+```
+
+NB that the cache has no expiry time
+(although it will be bypassed on subsequent runs if `DEBUG_CACHE isn't defined).
+You can clear the cache explicitly.
+
+```
+just clean-cache
+```
+
+
 ## Tests
 Run the tests with:
 ```

INSTALL.md

@@ -26,7 +26,7 @@ Code scanning alerts, Dependabot alerts, Issues, Metadata, Pull requests and Rep
 
 The `GITHUB_OS_TOKEN` is a fine-grained GitHub personal access token that is used for authenticating with the GitHub REST API.
 It is assigned to a single organisation and should have the following *read-only* permissions:
-* organisation permissions: codespaces
+* organisation permissions: organisation codespaces
 * *all repositories* owned by the organisation with the following permissions:
 Codespaces and Metadata
 

justfile

@@ -168,3 +168,7 @@ docker-build env="dev": _env
 docker-run env="dev" *args="": _env
     {{ just_executable() }} docker-build {{ env }}
     docker compose run --rm metrics-{{ env }} {{ args }}
+
+# See DEVELOPERS.md
+clean-cache:
+    rm -f github-cache.sqlite

metrics/github/client.py

@@ -1,13 +1,22 @@
 import json
+import os
 import textwrap
 
 import requests
 import requests.utils
+import requests_cache
 import structlog
 
 
 log = structlog.get_logger()
 
+# See DEVELOPERS.md
+if "DEBUG_CACHE" in os.environ:
+    requests_cache.install_cache(
+        "github-cache",
+        # Turn on caching for POST requests because that's what GraphQL uses
+        allowable_methods=("GET", "HEAD", "POST"),
+    )
 
 session = requests.Session()
 

requirements.prod.in

@@ -1,5 +1,6 @@
   greenlet
   requests
+  requests-cache
   slack-bolt
   sqlalchemy[postgresql_psycopgbinary]
   structlog

requirements.prod.txt

@@ -4,6 +4,16 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.prod.txt requirements.prod.in
 #
+attrs==24.2.0 \
+    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
+    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+    # via
+    #   cattrs
+    #   requests-cache
+cattrs==24.1.2 \
+    --hash=sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0 \
+    --hash=sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85
+    # via requests-cache
 certifi==2024.8.30 \
     --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
     --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
@@ -183,6 +193,10 @@ idna==3.10 \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
     # via requests
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via requests-cache
 psycopg[binary]==3.2.2 \
     --hash=sha256:8bad2e497ce22d556dac1464738cb948f8d6bab450d965cf1d8a8effd52412e0 \
     --hash=sha256:babf565d459d8f72fb65da5e211dd0b58a52c51e4e1fa9cadecff42d6b7619b2
@@ -256,11 +270,21 @@ psycopg-binary==3.2.2 \
 requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+    # via
+    #   -r requirements.prod.in
+    #   requests-cache
+requests-cache==1.2.1 \
+    --hash=sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603 \
+    --hash=sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1
     # via -r requirements.prod.in
 sentry-sdk==2.14.0 \
     --hash=sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d \
     --hash=sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4
     # via -r requirements.prod.in
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via url-normalize
 slack-bolt==1.20.1 \
     --hash=sha256:4657e592339797b9b804547a21e6b35dd8e2cd1eab676bfb23960660aae049fd \
     --hash=sha256:8fa26e72b0e55c18c1d34a73558e7fe2150bdc7c947de780b938fdb1d7e854fe
@@ -330,9 +354,14 @@ typing-extensions==4.12.2 \
     # via
     #   psycopg
     #   sqlalchemy
+url-normalize==1.4.3 \
+    --hash=sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2 \
+    --hash=sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed
+    # via requests-cache
 urllib3==2.2.3 \
     --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
     --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
     # via
     #   requests
+    #   requests-cache
     #   sentry-sdk