check_yaml: Use instructlab-schema package instead of git submodule

Signed-off-by: BJ Hargrave <hargrave@us.ibm.com>

.github/workflows/lint.yml

@@ -69,22 +69,34 @@ jobs:
             knowledge/**/*.yml
 
       - name: "Check changed YAML file contents"
-        if: ${{ steps.changed-files.outputs.any_changed == 'true' }}
+        if: ${{ fromJSON(steps.changed-files.outputs.any_changed) }}
         run: |
           scripts/check-yaml.py ${{ steps.changed-files.outputs.all_changed_files }}
         env:
-          SCHEMA_BASE: schema
           YAMLLINT_CONFIG: "{extends: relaxed, rules: {line-length: {max: 120}}}"
           TAXONOMY_FOLDERS: >-
             compositional_skills
             knowledge
 
+      - name: "Check all YAML file contents"
+        if: ${{ !fromJSON(steps.changed-files.outputs.any_changed) }}
+        run: |
+          read -ra folders <<< "${TAXONOMY_FOLDERS}"
+          # shellcheck disable=SC2046
+          scripts/check-yaml.py $(find "${folders[@]}" -name "qna.yaml")
+        env:
+          YAMLLINT_CONFIG: "{}" # No lint rules
+          SCHEMA_VERSION: 0 # use the schema version specified "version" key
+          TAXONOMY_FOLDERS: >-
+            compositional_skills
+            knowledge
+
       - name: "Save Pull Request number"
         if: ${{ (github.event_name == 'pull_request') && (github.repository == 'instructlab/taxonomy') }}
         run: |
           echo "${PULL_REQUEST_NUMBER}" > pull_request_number.txt
         env:
-          PULL_REQUEST_NUMBER: ${{ steps.changed-files.outputs.any_changed == 'true' && github.event.number || '0' }}
+          PULL_REQUEST_NUMBER: ${{ fromJSON(steps.changed-files.outputs.any_changed) && github.event.number || '0' }}
 
       - name: "Upload Pull Request number"
         if: ${{ (github.event_name == 'pull_request') && (github.repository == 'instructlab/taxonomy') }}

scripts/check-yaml.py

@@ -8,12 +8,14 @@
 import subprocess
 import sys
 from functools import cache, partial
+from importlib import resources
 from importlib.resources.abc import Traversable
 from pathlib import Path
 from typing import List, Mapping, Optional, Union
 
 # Third Party
 import yaml
+from instructlab.schema import schema_versions
 from jsonschema.protocols import Validator
 from jsonschema.validators import validator_for
 from referencing import Registry, Resource
@@ -29,24 +31,19 @@ def __init__(
         self,
         *,
         yaml_files: List[Path],
-        schema_base: Path,
         taxonomy_folders: List[str],
         yamllint_config: YamlLintConfig,
         schema_version: Optional[int] = None,
         message_format: Optional[str] = None,
     ) -> None:
         self.yaml_files = yaml_files
-        self.schema_base = schema_base
         self.taxonomy_folders = taxonomy_folders
         self.yamllint_config = yamllint_config
+        self.schema_base = resources.files("instructlab.schema")
         if schema_version is None:
-            schema_versions = sorted(
-                int(v.name[1:])
-                for v in self.schema_base.glob("v*")
-                if v.name[1:].isdigit()
-            )
-            if schema_versions:
-                schema_version = schema_versions[-1]
+            versions = schema_versions()
+            if versions:
+                schema_version = int(versions[-1].name[1:])
         self.schema_version = schema_version
         if message_format is None or message_format == "auto":
             message_format = (
@@ -58,7 +55,7 @@ def __init__(
         self.exit_code: int = 0
 
     @cache
-    def _load_schema(self, path: Union[Path, Traversable]) -> Resource:
+    def _load_schema(self, path: Traversable) -> Resource:
         try:
             contents = json.loads(path.read_text(encoding="utf-8"))
             resource = Resource.from_contents(
@@ -68,7 +65,7 @@ def _load_schema(self, path: Union[Path, Traversable]) -> Resource:
             raise NoSuchResource(ref=str(path)) from e
         return resource
 
-    def _retrieve(self, schemas_path: Union[Path, Traversable], uri: URI) -> Resource:
+    def _retrieve(self, schemas_path: Traversable, uri: URI) -> Resource:
         path = schemas_path.joinpath(uri)
         return self._load_schema(path)
 
@@ -279,17 +276,6 @@ def cli() -> int:
             "TAXONOMY_FOLDERS", "compositional_skills knowledge"
         ).split(),
     )
-    parser.add_argument(
-        "-s",
-        "--schema-base",
-        help="""
-            The base directory of the Taxonomy schema files.
-            Alternately, the SCHEMA_BASE environment variable can be used
-            to specify the base directory.
-            """,
-        default=os.environ.get("SCHEMA_BASE", _find_schema_base()),
-        type=Path,
-    )
     parser.add_argument(
         "-v",
         "--schema-version",
@@ -337,23 +323,12 @@ def cli() -> int:
         yaml_files=args.yaml_file,
         taxonomy_folders=args.taxonomy_folders,
         yamllint_config=args.yamllint_config,
-        schema_base=args.schema_base,
         schema_version=args.schema_version,
         message_format=args.message_format,
     )
     exit_code = check_yaml.check()
     return exit_code
 
 
-def _find_schema_base() -> Path:
-    for parent in Path(sys.argv[0]).parents:
-        candidate = parent.joinpath("schema")
-        if os.path.isdir(candidate):
-            return candidate
-        if os.path.exists(parent.joinpath(".git")):
-            break
-    return Path.cwd().joinpath("schema")
-
-
 if __name__ == "__main__":
     sys.exit(cli())

scripts/requirements.txt

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
+instructlab-schema @ git+https://github.com/bjhargrave/instructlab_schema@python-package
 jsonschema>=4.21.1,<5.0.0
 PyYAML>=6.0.1,<7.0.0
 yamllint>=1.35.1,<2.0.0