ENH: use git ls-files instead of glob() (#329)

* ENH: ignore untracked files in cspell check
* ENH: use `git ls-files` instead of `glob()`
* FIX: wrap readthedocs pip install line

.cspell.json

@@ -22,6 +22,8 @@
   "ignorePaths": [
     "**/*.rst_t",
     "**/.cspell.json",
+    "*.ico",
+    "*.rst_t",
     ".editorconfig",
     ".gitignore",
     ".gitpod.*",
@@ -33,6 +35,7 @@
     "docs/conf.py",
     "labels/*.toml",
     "pyproject.toml",
+    "pyrightconfig.json",
     "tox.ini",
     "typings"
   ],
@@ -85,6 +88,7 @@
     "noreply",
     "orcid",
     "pandoc",
+    "pathspec",
     "prebuilds",
     "precommit",
     "prereleased",

.pre-commit-config.yaml

@@ -78,8 +78,7 @@ repos:
     rev: v0.23.1
     hooks:
       - id: toml-sort
-        args:
-          - --in-place
+        args: [--in-place]
         exclude: (?x)^(labels/.*\.toml)$
 
   - repo: https://github.com/streetsidesoftware/cspell-cli

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "html2text",
     "ini2toml",
     "nbformat",
+    "pathspec",
     "pip-tools",
     "ruamel.yaml", # better YAML dumping
     "tomlkit", # preserve original TOML formatting

src/compwa_policy/check_dev_files/cspell.py

@@ -8,12 +8,12 @@
 
 import json
 import os
-from glob import glob
 from typing import TYPE_CHECKING, Any, Iterable, Sequence
 
 from compwa_policy.errors import PrecommitError
 from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, rename_file, vscode
 from compwa_policy.utilities.executor import Executor
+from compwa_policy.utilities.match import filter_patterns
 from compwa_policy.utilities.precommit.struct import Hook, Repo
 from compwa_policy.utilities.readme import add_badge, remove_badge
 
@@ -157,9 +157,7 @@ def __get_expected_content(config: dict, section: str, *, extend: bool = False)
         return expected_section_content
     if isinstance(expected_section_content, list):
         if section == "ignorePaths":
-            expected_section_content = [
-                p for p in expected_section_content if glob(p, recursive=True)
-            ]
+            expected_section_content = filter_patterns(expected_section_content)
         if not extend:
             return __sort_section(expected_section_content, section)
         expected_section_content_set = set(expected_section_content)

src/compwa_policy/check_dev_files/github_labels.py

@@ -7,10 +7,12 @@
 from __future__ import annotations
 
 import os
-import pathlib
+from functools import lru_cache
+from pathlib import Path
 
 from compwa_policy.errors import PrecommitError
 from compwa_policy.utilities import CONFIG_PATH
+from compwa_policy.utilities.match import filter_files
 
 __LABELS_CONFIG_FILE = "labels.toml"
 
@@ -39,7 +41,7 @@ def main() -> None:
         raise PrecommitError(msg)
 
 
-def _check_has_labels_requirement(path: pathlib.Path) -> bool:
+def _check_has_labels_requirement(path: Path) -> bool:
     with open(path) as stream:
         lines = stream.readlines()
     for line in lines:
@@ -49,12 +51,15 @@ def _check_has_labels_requirement(path: pathlib.Path) -> bool:
     return False
 
 
-def _get_requirement_files() -> list[pathlib.Path]:
-    return [
-        *pathlib.Path(".").glob("**/requirements*.in"),
-        *pathlib.Path(".").glob("**/requirements*.txt"),
-        *pathlib.Path(".").glob(str(CONFIG_PATH.setup_cfg)),
+@lru_cache(maxsize=1)
+def _get_requirement_files() -> list[Path]:
+    patterns = [
+        "**/requirements*.in",
+        "**/requirements*.txt",
+        str(CONFIG_PATH.setup_cfg),
     ]
+    filenames = filter_files(patterns)
+    return [Path(file) for file in filenames]
 
 
 def _get_package_name(line: str) -> str:
@@ -72,7 +77,7 @@ def _remove_all_labels_requirement() -> None:
         _remove_labels_requirement(file)
 
 
-def _remove_labels_requirement(path: pathlib.Path) -> None:
+def _remove_labels_requirement(path: Path) -> None:
     with open(path) as stream:
         original_lines = stream.readlines()
     with open(path, "w") as stream:

src/compwa_policy/check_dev_files/readthedocs.py

@@ -7,7 +7,7 @@
 from typing import IO, TYPE_CHECKING, cast
 
 from ruamel.yaml.comments import CommentedMap
-from ruamel.yaml.scalarstring import DoubleQuotedScalarString
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString, LiteralScalarString
 
 from compwa_policy.errors import PrecommitError
 from compwa_policy.utilities import CONFIG_PATH, get_nested_dict
@@ -95,7 +95,7 @@ def __get_install_steps(python_version: PythonVersion) -> list[str]:
         install_statement = f"{pip_install} -c {constraints_file} -e .[doc]"
     return [
         "curl -LsSf https://astral.sh/uv/install.sh | sh",
-        install_statement,
+        LiteralScalarString(install_statement),
     ]
 
 

src/compwa_policy/check_dev_files/toml.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import shutil
-from glob import glob
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -13,6 +12,7 @@
 from compwa_policy.errors import PrecommitError
 from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, vscode
 from compwa_policy.utilities.executor import Executor
+from compwa_policy.utilities.match import filter_patterns
 from compwa_policy.utilities.precommit.struct import Hook, Repo
 from compwa_policy.utilities.pyproject import ModifiablePyproject
 from compwa_policy.utilities.toml import to_toml_array
@@ -72,16 +72,17 @@ def _update_tomlsort_hook(precommit: ModifiablePrecommit) -> None:
         rev="",
         hooks=[Hook(id="toml-sort", args=YAML(typ="rt").load("[--in-place]"))],
     )
-    excludes = []
-    if glob("labels/*.toml"):
-        excludes.append(r"labels/.*\.toml")
-    if glob("labels*.toml"):
-        excludes.append(r"labels.*\.toml")
-    if any(glob(f"**/{f}.toml", recursive=True) for f in ("Manifest", "Project")):
-        excludes.append(r".*(Manifest|Project)\.toml")
+    excludes = filter_patterns([
+        "**/Manifest.toml",
+        "**/Project.toml",
+        "labels*.toml",
+        "labels/*.toml",
+    ])
     if excludes:
-        excludes = sorted(excludes, key=str.lower)
-        expected_hook["hooks"][0]["exclude"] = "(?x)^(" + "|".join(excludes) + ")$"
+        regex_excludes = sorted(_to_regex(r) for r in excludes)
+        expected_hook["hooks"][0]["exclude"] = (
+            "(?x)^(" + "|".join(regex_excludes) + ")$"
+        )
     precommit.update_single_hook_repo(expected_hook)
 
 
@@ -102,10 +103,11 @@ def _update_taplo_config() -> None:
         raise PrecommitError(msg)
     with open(template_path) as f:
         expected = tomlkit.load(f)
-    excludes: list[str] = [p for p in expected["exclude"] if glob(p, recursive=True)]  # type: ignore[union-attr]
+
+    excludes = filter_patterns(expected["exclude"])  # type:ignore[arg-type]
     if excludes:
-        excludes = sorted(excludes, key=str.lower)
-        expected["exclude"] = to_toml_array(excludes, enforce_multiline=True)
+        sorted_excludes = sorted(excludes, key=str.lower)
+        expected["exclude"] = to_toml_array(sorted_excludes, enforce_multiline=True)
     else:
         del expected["exclude"]
     with open(CONFIG_PATH.taplo) as f:
@@ -133,3 +135,12 @@ def _update_vscode_extensions() -> None:
     with Executor() as do:
         do(vscode.add_extension_recommendation, "tamasfe.even-better-toml")
         do(vscode.remove_extension_recommendation, "bungcip.better-toml", unwanted=True)
+
+
+def _to_regex(glob: str) -> str:
+    r"""Convert glob pattern to regex.
+
+    >>> _to_regex("**/*.toml")
+    '.*/.*\\.toml'
+    """
+    return glob.replace("**", "*").replace(".", r"\.").replace("*", r".*")

src/compwa_policy/check_dev_files/update_pip_constraints.py

@@ -8,7 +8,6 @@
 from __future__ import annotations
 
 import sys
-from glob import glob
 from typing import TYPE_CHECKING
 
 from compwa_policy.check_dev_files.github_workflows import (
@@ -18,6 +17,7 @@
 from compwa_policy.errors import PrecommitError
 from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH
 from compwa_policy.utilities.executor import Executor
+from compwa_policy.utilities.match import filter_patterns
 from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml
 
 if TYPE_CHECKING:
@@ -74,9 +74,9 @@ def overwrite_workflow(workflow_file: str) -> None:
         if frequency == "outsource":
             del expected_data["on"]["schedule"]
         else:
-            paths: list[str] = expected_data["on"]["pull_request"]["paths"]
+            paths = filter_patterns(expected_data["on"]["pull_request"]["paths"])
+            expected_data["on"]["pull_request"]["paths"] = paths
             expected_data["on"]["schedule"][0]["cron"] = _to_cron_schedule(frequency)
-            expected_data["on"]["pull_request"]["paths"] = [p for p in paths if glob(p)]
         workflow_path = CONFIG_PATH.github_workflow_dir / workflow_file
         if not workflow_path.exists():
             update_workflow(yaml, expected_data, workflow_path)

src/compwa_policy/utilities/match.py

@@ -0,0 +1,80 @@
+"""Functions for checking whether files exist on disk."""
+
+from __future__ import annotations
+
+import subprocess  # noqa: S404
+
+from pathspec import PathSpec
+from pathspec.patterns import GitWildMatchPattern
+
+
+def filter_files(patterns: list[str], files: list[str] | None = None) -> list[str]:
+    """Filter filenames that match certain patterns.
+
+    If :code:`files` is not supplied, get the files with :func:`git_ls_files`.
+
+    >>> filter_files(["**/*.json", "**/*.txt"], ["a/b/file.json", "file.yaml"])
+    ['a/b/file.json']
+    """
+    if files is None:
+        files = git_ls_files(untracked=True)
+    return [file for file in files if matches_patterns(file, patterns)]
+
+
+def filter_patterns(patterns: list[str], files: list[str] | None = None) -> list[str]:
+    """Filter patterns that match files.
+
+    If :code:`files` is not supplied, get the files with :func:`git_ls_files`.
+
+    >>> filter_patterns(["**/*.json", "**/*.txt"], ["file.json", "file.yaml"])
+    ['**/*.json']
+    """
+    if files is None:
+        files = git_ls_files(untracked=True)
+    return [pattern for pattern in patterns if matches_files(pattern, files)]
+
+
+def git_ls_files(untracked: bool = False) -> list[str]:
+    """Get the tracked and untracked files, but excluding files in .gitignore."""
+    output = subprocess.check_output([  # noqa: S603, S607
+        "git",
+        "ls-files",
+    ]).decode("utf-8")
+    tracked_files = output.splitlines()
+    if untracked:
+        output = subprocess.check_output([  # noqa: S603, S607
+            "git",
+            "ls-files",
+            "--others",
+            "--exclude-standard",
+        ]).decode("utf-8")
+        return tracked_files + output.splitlines()
+    return tracked_files
+
+
+def matches_files(pattern: str, files: list[str]) -> bool:
+    """Use git wild-match patterns to match a filename.
+
+    >>> matches_files("**/*.json", [".cspell.json"])
+    True
+    >>> matches_files("**/*.json", ["some/random/path/.cspell.json"])
+    True
+    >>> matches_files("*/*.json", ["some/random/path/.cspell.json"])
+    False
+    """
+    spec = PathSpec.from_lines(GitWildMatchPattern, [pattern])
+    return any(spec.match_file(file) for file in files)
+
+
+def matches_patterns(filename: str, patterns: list[str]) -> bool:
+    """Use git wild-match patterns to match a filename.
+
+    >>> matches_patterns(".cspell.json", patterns=["**/*.json"])
+    True
+    >>> matches_patterns("some/random/path/.cspell.json", patterns=["**/*.json"])
+    True
+    >>> matches_patterns("some/random/path/.cspell.json", patterns=["*/*.json"])
+    False
+    """
+    spec = PathSpec.from_lines(GitWildMatchPattern, patterns)
+    return spec.match_file(filename)

tests/check_dev_files/readthedocs/.readthedocs-good.yml

@@ -6,4 +6,5 @@ build:
   jobs:
     post_install:
       - curl -LsSf https://astral.sh/uv/install.sh | sh
-      - /home/docs/.cargo/bin/uv pip install --system -e .[doc]
+      - |-
+        /home/docs/.cargo/bin/uv pip install --system -e .[doc]