schema package: Build a python package for the schema

This will allow schema users to pip install the schema instead of
using something like git submodules to place the schema files at some
local path.

Signed-off-by: BJ Hargrave <hargrave@us.ibm.com>

.github/dependabot.yml

@@ -13,9 +13,3 @@ updates:
     directory: "/.github/workflows"
     schedule:
       interval: "daily"
-
-  # Maintain dependencies for Python scripts
-  - package-ecosystem: "pip"
-    directory: "/.github/scripts"
-    schedule:
-      interval: "daily"

.github/workflows/actionlint.yml

@@ -38,7 +38,6 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         with:
           fetch-depth: 0
-          submodules: true
 
       - name: "Download actionlint"
         run: |

.github/workflows/docs.yml

@@ -40,7 +40,6 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         with:
           fetch-depth: 0
-          submodules: true
       - name: "Check Markdown documents"
         uses: DavidAnson/markdownlint-cli2-action@b4c9feab76d8025d1e83c653fa3990936df0e6c8 # v16.0.0
         with:

.github/workflows/lint.yml

@@ -1,24 +1,29 @@
 # SPDX-License-Identifier: Apache-2.0
 
-name: Lint Schema
+name: Lint
 
 on:
-  workflow_dispatch:
   push:
     branches:
-      - main
+      - "main"
     paths:
-      - 'v*/**/*.json'
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
       - '.github/workflows/lint.yml' # This workflow
-      - '.github/scripts/**' # Scripts used by this workflow
 
   pull_request:
     branches:
-      - main
+      - "main"
     paths:
-      - 'v*/**/*.json'
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
       - '.github/workflows/lint.yml' # This workflow
-      - '.github/scripts/**' # Scripts used by this workflow
 
 env:
   LC_ALL: en_US.UTF-8
@@ -33,6 +38,25 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
+    name: "${{ matrix.lint.name }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        lint:
+          - name: "jsonschema"
+            commands: |
+              tox -e jsonschema
+          - name: "ruff"
+            commands: |
+              tox -e ruff -- check
+          - name: "pylint"
+            commands: |
+              echo "::add-matcher::.github/workflows/matchers/pylint.json"
+              tox -e lint
+          - name: "mypy"
+            commands: |
+              echo "::add-matcher::.github/workflows/matchers/mypy.json"
+              tox -e mypy
     steps:
       - name: "Harden Runner"
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
@@ -49,24 +73,13 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: "Install Python Packages"
+      - name: "Install tox"
         run: |
-          pip install -r .github/scripts/requirements.txt
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh
 
-      - name: "Find changed schema files"
-        id: changed-files
-        uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2
-        with:
-          files: |
-            v*/**/*.json
-
-      - name: "Check changed schema file contents"
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          check-jsonschema --verbose --schemafile https://json-schema.org/draft/2020-12/schema ${{ steps.changed-files.outputs.all_changed_files }}
-
-      - name: "Check all schema file contents"
-        if: steps.changed-files.outputs.any_changed != 'true'
+      - name: "${{ matrix.lint.name }}"
         run: |
-          # shellcheck disable=SC2046
-          check-jsonschema --verbose --schemafile https://json-schema.org/draft/2020-12/schema $(find v* -name "*.json")
+          ${{ matrix.lint.commands }}
+        env:
+          RUFF_OUTPUT_FORMAT: github

.github/workflows/matchers/mypy.json

@@ -0,0 +1,16 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "mypy",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):\\s(error|warning):\\s(.+)$",
+          "file": 1,
+          "line": 2,
+          "severity": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.github/workflows/matchers/pylint.json

@@ -0,0 +1,32 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "pylint-error",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):(\\d+):\\s(([EF]\\d{4}):\\s.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4,
+          "code": 5
+        }
+      ]
+    },
+    {
+      "owner": "pylint-warning",
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^(.+):(\\d+):(\\d+):\\s(([CRW]\\d{4}):\\s.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4,
+          "code": 5
+        }
+      ]
+    }
+  ]
+}

.github/workflows/pypi.yml

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build, test, and upload PyPI package
+
+on:
+    push:
+        branches:
+            - "main"
+        tags:
+            - "v*"
+    pull_request:
+        branches:
+            - "main"
+    release:
+        types:
+            - published
+
+env:
+    LC_ALL: en_US.UTF-8
+
+defaults:
+    run:
+        shell: bash
+
+permissions:
+    contents: read
+
+jobs:
+    # Create and verify release artifacts
+    # - build source dist (tar ball) and wheel
+    # - validate artifacts with various tools
+    # - upload artifacts to GHA
+    build-package:
+        name: Build and check packages
+        runs-on: ubuntu-latest
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+
+            - name: "Checkout"
+              uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+
+            - name: "Build and Inspect"
+              uses: hynek/build-and-inspect-python-package@b4fc3f6ba2b3da04f09659be99e2a29fb6146a61 # v2.6.0
+
+    # push to Test PyPI on
+    # - a new GitHub release is published
+    # - a PR is merged into main branch
+    publish-test-pypi:
+        name: Publish packages to test.pypi.org
+        # environment: publish-test-pypi
+        if: ${{ false && (github.repository_owner == 'instructlab') && ((github.event.action == 'published') || ((github.event_name == 'push') && (github.ref == 'refs/heads/main'))) }}
+        permissions:
+            contents: read
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Upload to Test PyPI"
+              uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    # push to Production PyPI on
+    # - a new GitHub release is published
+    publish-pypi:
+        name: Publish release to pypi.org
+        # environment: publish-pypi
+        if: ${{ false && (github.repository_owner == 'instructlab') && (github.event.action == 'published') }}
+        permissions:
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+            # allow gh release upload
+            contents: write
+
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@f086349bfa2bd1361f7909c78558e816508cdc10 # v2.8.0
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Sigstore sign package"
+              uses: sigstore/gh-action-sigstore-python@61f6a500bbfdd9a2a339cf033e5421951fbc1cd2 # v2.1.1
+              with:
+                  inputs: |
+                      ./dist/*.tar.gz
+                      ./dist/*.whl
+
+            - name: "Upload artifacts and signatures to GitHub release"
+              run: |
+                  gh release upload '${{ github.ref_name }}' dist/* --repo '${{ github.repository }}'
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            # PyPI does not accept .sigstore artifacts and
+            # gh-action-pypi-publish has no option to ignore them.
+            - name: "Remove sigstore signatures before uploading to PyPI"
+              run: |
+                  rm ./dist/*.sigstore
+
+            - name: "Upload to PyPI"
+              uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14

.github/workflows/test.yml

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
+      - '.github/workflows/test.yml' # This workflow
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - '**.py'
+      - 'src/instructlab/schema/v*/**/*.json'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - 'scripts/**'
+      - '.github/workflows/test.yml' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    strategy:
+      matrix:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        platform:
+          - "ubuntu-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: "Checkout"
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+
+      - name: "Setup Python ${{ matrix.python }}"
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: "Install tox"
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh
+
+      - name: "Unit tests"
+        run: |
+          tox