From 1e03f863d6d90f0a174fc0dc6e8ef58df2f876fa Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:13:30 +0200 Subject: [PATCH 1/8] use scalar instead of redoc for the docs --- launcher/commands/doc.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/launcher/commands/doc.py b/launcher/commands/doc.py index 95f44e97..250a4fc4 100644 --- a/launcher/commands/doc.py +++ b/launcher/commands/doc.py @@ -1,11 +1,57 @@ import subprocess import os import json +import jinja2 from launcher.config import check_config from launcher.skyportal import patch as patch_skyportal +def patch_api_doc_template(): + """Patch the API documentation template with the OpenAPI spec. + + This function reads the OpenAPI specification from the openapi.json file, + populates it with server information from the configuration, and renders + a HTML documentation page from https://github.com/scalar/scalar + using a Jinja2 template. + + Raises: + ValueError: If the server information is not valid. + """ + # open the openapi.json file generated by build-spec.py + from baselayer.app.env import load_env + + _, cfg = load_env() + + with open("openapi.json") as f: + openapi_spec = json.load(f) + + # populate the OpenAPI spec with (optional) server information + servers = cfg.get("docs.servers", []) + if servers is None: + servers = [] + if not isinstance(servers, list): + raise ValueError("API servers must be a list.") + + for server in servers: + if not all(k in server for k in ("url", "description")): + raise ValueError("Each server must have 'url' and 'description' keys.") + openapi_spec["servers"] = servers + + # Create a Jinja2 template environment + template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath="./")) + + # Load the template file + template = template_env.get_template("doc/openapi.html.template") + + # Render the template with the OpenAPI spec + output = template.render(openapi_spec=json.dumps(openapi_spec, indent=2)) + + # Write the output to a new HTML file + with open("../doc/_build/html/api.html", "w") as f: + f.write(output) + + def doc(yes: bool = False, upload: bool = False): """Build the documentation @@ -46,6 +92,8 @@ def doc(yes: bool = False, upload: bool = False): with open("skyportal/openapi.json", "w") as f: json.dump(spec.to_dict(), f) + patch_api_doc_template() + subprocess.run( [ "npx", From 7403dc97bca82c2129ecd39bacd3583cc3ded04e Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:21:44 +0200 Subject: [PATCH 2/8] cache sncosmo and dustmap files --- .github/workflows/build-test-doc_upload.yaml | 34 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-doc_upload.yaml b/.github/workflows/build-test-doc_upload.yaml index d5e8a4b5..b2dfbeb7 100644 --- a/.github/workflows/build-test-doc_upload.yaml +++ b/.github/workflows/build-test-doc_upload.yaml @@ -5,14 +5,22 @@ on: branches: - main pull_request: + types: [opened, reopened, synchronize, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: write jobs: build-test-doc: runs-on: ubuntu-latest timeout-minutes: 20 - steps: + if: github.repository_owner == 'fritz' && github.event.pull_request.draft == false + + steps: - uses: actions/checkout@v4 with: submodules: recursive @@ -25,18 +33,38 @@ jobs: with: node-version: 20 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: | ~/.npm key: ${{ runner.os }}-npm-${{ hashFiles('package.json') }} - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: | ~/.cache/pip key: ${{ runner.os }}-${{ hashFiles('**/requirements*.txt') }} + # caching dustmap & sncosmo files is dependent on their corresponding + # python package versions, so we use that as the cache key + - uses: actions/cache@v4 + with: + path: | + persistentdata/dustmap/sfd + key: ${{ runner.os }}-dustmap-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-dustmap-${{ hashFiles('**/requirements*.txt') }} + ${{ runner.os }}-dustmap- + + - uses: actions/cache@v4 + with: + path: | + ~/.astropy/cache/sncosmo/bandpasses + key: ${{ runner.os }}-sncosmo-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-sncosmo-${{ hashFiles('**/requirements*.txt') }} + ${{ runner.os }}-sncosmo- + - name: Install system dependencies run: | sudo apt update -y && sudo apt install -y libcurl4-gnutls-dev libgnutls28-dev From e56e8732b1d0bbe6eaf569d7bff78bb3b73f1f57 Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:24:36 +0200 Subject: [PATCH 3/8] fix some paths --- launcher/commands/doc.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/launcher/commands/doc.py b/launcher/commands/doc.py index 250a4fc4..a76c1316 100644 --- a/launcher/commands/doc.py +++ b/launcher/commands/doc.py @@ -42,7 +42,7 @@ def patch_api_doc_template(): template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath="./")) # Load the template file - template = template_env.get_template("doc/openapi.html.template") + template = template_env.get_template("skyportal/doc/openapi.html.template") # Render the template with the OpenAPI spec output = template.render(openapi_spec=json.dumps(openapi_spec, indent=2)) @@ -89,29 +89,12 @@ def doc(yes: bool = False, upload: bool = False): "servers": [{"url": "https://fritz.science"}], }, ) - with open("skyportal/openapi.json", "w") as f: + with open("openapi.json", "w") as f: json.dump(spec.to_dict(), f) patch_api_doc_template() - subprocess.run( - [ - "npx", - "redoc-cli@0.13.21", - "bundle", - "openapi.json", - "--title", - "Fritz API docs", - "--cdn", - "--options.theme.logo.gutter", - "2rem", - "-o", - "../doc/_build/html/api.html", - ], - check=True, - cwd="skyportal", - ) - os.remove("skyportal/openapi.json") + os.remove("openapi.json") if upload: subprocess.run( From 3b97a0a7647401a463cec3441087570c6af5e47b Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:25:57 +0200 Subject: [PATCH 4/8] fix repo owner name --- .github/workflows/build-test-doc_upload.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-doc_upload.yaml b/.github/workflows/build-test-doc_upload.yaml index b2dfbeb7..a5a456c0 100644 --- a/.github/workflows/build-test-doc_upload.yaml +++ b/.github/workflows/build-test-doc_upload.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.repository_owner == 'fritz' && github.event.pull_request.draft == false + if: github.repository_owner == 'skyportal' && github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 From afe6f577e2c8d1e0f38d6aa642cfc160ddcd4dec Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:26:50 +0200 Subject: [PATCH 5/8] try to comment out pull_request conditions --- .github/workflows/build-test-doc_upload.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-doc_upload.yaml b/.github/workflows/build-test-doc_upload.yaml index a5a456c0..bcaefa88 100644 --- a/.github/workflows/build-test-doc_upload.yaml +++ b/.github/workflows/build-test-doc_upload.yaml @@ -5,7 +5,7 @@ on: branches: - main pull_request: - types: [opened, reopened, synchronize, ready_for_review] + # types: [opened, reopened, synchronize, ready_for_review] concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 20acb9ecfa6083c966bce74d70de2d49f3040009 Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:27:27 +0200 Subject: [PATCH 6/8] comment out repo owner check --- .github/workflows/build-test-doc_upload.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-doc_upload.yaml b/.github/workflows/build-test-doc_upload.yaml index bcaefa88..daf629f9 100644 --- a/.github/workflows/build-test-doc_upload.yaml +++ b/.github/workflows/build-test-doc_upload.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.repository_owner == 'skyportal' && github.event.pull_request.draft == false + # if: github.repository_owner == 'skyportal' && github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 From 971b81e71068e3d4d2c19242c79854ff83216b0d Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:31:31 +0200 Subject: [PATCH 7/8] fix the repo owner --- .github/workflows/build-test-doc_upload.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-doc_upload.yaml b/.github/workflows/build-test-doc_upload.yaml index daf629f9..cb6082e0 100644 --- a/.github/workflows/build-test-doc_upload.yaml +++ b/.github/workflows/build-test-doc_upload.yaml @@ -5,7 +5,7 @@ on: branches: - main pull_request: - # types: [opened, reopened, synchronize, ready_for_review] + types: [opened, reopened, synchronize, ready_for_review] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - # if: github.repository_owner == 'skyportal' && github.event.pull_request.draft == false + if: github.repository_owner == 'fritz-marshal' && github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 From e96d5517ace05868c8da6987cacf2f0e95213e9f Mon Sep 17 00:00:00 2001 From: Theodlz Date: Mon, 7 Oct 2024 23:35:08 +0200 Subject: [PATCH 8/8] try to edit path --- launcher/commands/doc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launcher/commands/doc.py b/launcher/commands/doc.py index a76c1316..3db9e20f 100644 --- a/launcher/commands/doc.py +++ b/launcher/commands/doc.py @@ -48,7 +48,8 @@ def patch_api_doc_template(): output = template.render(openapi_spec=json.dumps(openapi_spec, indent=2)) # Write the output to a new HTML file - with open("../doc/_build/html/api.html", "w") as f: + os.makedirs("./doc/_build/html", exist_ok=True) + with open("./doc/_build/html/api.html", "w") as f: f.write(output)