Merge pull request #27 from bjhargrave/python-package

schema package: Build a python package with the schema

.github/dependabot.yml

@@ -13,9 +13,3 @@ updates:
     directory: "/.github/workflows"
     schedule:
       interval: "daily"
-
-  # Maintain dependencies for Python scripts
-  - package-ecosystem: "pip"
-    directory: "/.github/scripts"
-    schedule:
-      interval: "daily"

.github/workflows/actionlint.yml

@@ -38,7 +38,6 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           fetch-depth: 0
-          submodules: true
 
       - name: "Download actionlint"
         run: |

.github/workflows/docs.yml

@@ -40,7 +40,6 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           fetch-depth: 0
-          submodules: true
       - name: "Check Markdown documents"
         uses: DavidAnson/markdownlint-cli2-action@b4c9feab76d8025d1e83c653fa3990936df0e6c8 # v16.0.0
         with:

.github/workflows/lint.yml

@@ -1,24 +1,29 @@
 # SPDX-License-Identifier: Apache-2.0
 
-name: Lint Schema
+name: Lint
 
 on:
-  workflow_dispatch:
   push:
     branches:
-      - main
+      - "main"
     paths:
-      - 'v*/**/*.json'
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
       - '.github/workflows/lint.yml' # This workflow
-      - '.github/scripts/**' # Scripts used by this workflow
 
   pull_request:
     branches:
-      - main
+      - "main"
     paths:
-      - 'v*/**/*.json'
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
       - '.github/workflows/lint.yml' # This workflow
-      - '.github/scripts/**' # Scripts used by this workflow
 
 env:
   LC_ALL: en_US.UTF-8
@@ -33,6 +38,25 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
+    name: "${{ matrix.lint.name }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        lint:
+          - name: "jsonschema"
+            commands: |
+              tox -e jsonschema
+          - name: "ruff"
+            commands: |
+              tox -e ruff -- check
+          - name: "pylint"
+            commands: |
+              echo "::add-matcher::.github/workflows/matchers/pylint.json"
+              tox -e pylint
+          - name: "mypy"
+            commands: |
+              echo "::add-matcher::.github/workflows/matchers/mypy.json"
+              tox -e mypy
     steps:
       - name: "Harden Runner"
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
@@ -49,24 +73,13 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: "Install Python Packages"
+      - name: "Install tox"
         run: |
-          pip install -r .github/scripts/requirements.txt
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh
 
-      - name: "Find changed schema files"
-        id: changed-files
-        uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2
-        with:
-          files: |
-            v*/**/*.json
-
-      - name: "Check changed schema file contents"
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          check-jsonschema --verbose --schemafile https://json-schema.org/draft/2020-12/schema ${{ steps.changed-files.outputs.all_changed_files }}
-
-      - name: "Check all schema file contents"
-        if: steps.changed-files.outputs.any_changed != 'true'
+      - name: "${{ matrix.lint.name }}"
         run: |
-          # shellcheck disable=SC2046
-          check-jsonschema --verbose --schemafile https://json-schema.org/draft/2020-12/schema $(find v* -name "*.json")
+          ${{ matrix.lint.commands }}
+        env:
+          RUFF_OUTPUT_FORMAT: github

.github/workflows/matchers/mypy.json

@@ -0,0 +1,16 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "mypy",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):\\s(error|warning):\\s(.+)$",
+          "file": 1,
+          "line": 2,
+          "severity": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.github/workflows/matchers/pylint.json

@@ -0,0 +1,32 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "pylint-error",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):(\\d+):\\s(([EF]\\d{4}):\\s.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4,
+          "code": 5
+        }
+      ]
+    },
+    {
+      "owner": "pylint-warning",
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):(\\d+):\\s(([CRW]\\d{4}):\\s.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4,
+          "code": 5
+        }
+      ]
+    }
+  ]
+}

.github/workflows/pypi.yml

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build, test, and upload PyPI package
+
+on:
+    push:
+        branches:
+            - "main"
+        tags:
+            - "v*"
+    pull_request:
+        branches:
+            - "main"
+    release:
+        types:
+            - published
+
+env:
+    LC_ALL: en_US.UTF-8
+
+defaults:
+    run:
+        shell: bash
+
+permissions:
+    contents: read
+
+jobs:
+    # Create and verify release artifacts
+    # - build source dist (tar ball) and wheel
+    # - validate artifacts with various tools
+    # - upload artifacts to GHA
+    build-package:
+        name: Build and check packages
+        runs-on: ubuntu-latest
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+
+            - name: "Checkout"
+              uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+
+            - name: "Build and Inspect"
+              uses: hynek/build-and-inspect-python-package@b4fc3f6ba2b3da04f09659be99e2a29fb6146a61 # v2.6.0
+
+    # push to Test PyPI on
+    # - a new GitHub release is published
+    # - a PR is merged into main branch
+    publish-test-pypi:
+        name: Publish packages to test.pypi.org
+        # environment: publish-test-pypi
+        if: ${{ (github.repository_owner == 'instructlab') && ((github.event.action == 'published') || ((github.event_name == 'push') && (github.ref == 'refs/heads/main'))) }}
+        permissions:
+            contents: read
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Upload to Test PyPI"
+              uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    # push to Production PyPI on
+    # - a new GitHub release is published
+    publish-pypi:
+        name: Publish release to pypi.org
+        # environment: publish-pypi
+        if: ${{ (github.repository_owner == 'instructlab') && (github.event.action == 'published') }}
+        permissions:
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+            # allow gh release upload
+            contents: write
+
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Sigstore sign package"
+              uses: sigstore/gh-action-sigstore-python@61f6a500bbfdd9a2a339cf033e5421951fbc1cd2 # v2.1.1
+              with:
+                  inputs: |
+                      ./dist/*.tar.gz
+                      ./dist/*.whl
+
+            - name: "Upload artifacts and signatures to GitHub release"
+              run: |
+                  gh release upload '${{ github.ref_name }}' dist/* --repo '${{ github.repository }}'
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            # PyPI does not accept .sigstore artifacts and
+            # gh-action-pypi-publish has no option to ignore them.
+            - name: "Remove sigstore signatures before uploading to PyPI"
+              run: |
+                  rm ./dist/*.sigstore
+
+            - name: "Upload to PyPI"
+              uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14

.github/workflows/test.yml

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
+      - '.github/workflows/test.yml' # This workflow
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
+      - '.github/workflows/test.yml' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    strategy:
+      matrix:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        platform:
+          - "ubuntu-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: "Checkout"
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          fetch-depth: 0
+
+      - name: "Setup Python ${{ matrix.python }}"
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: "Install tox"
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh
+
+      - name: "Unit tests"
+        run: |
+          tox