diff --git a/.cspell.json b/.cspell.json index a12a51dc..9395eb4b 100644 --- a/.cspell.json +++ b/.cspell.json @@ -88,6 +88,7 @@ "prereleased", "prettierignore", "prettierrc", + "pyenv", "pyright", "pyupgrade", "redeboer", diff --git a/.gitpod.yml b/.gitpod.yml index c3e87fa6..6cb70a6b 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,5 @@ tasks: + - init: pyenv local 3.8 - init: pip install -e .[dev] github: diff --git a/src/repoma/check_dev_files/__init__.py b/src/repoma/check_dev_files/__init__.py index 27bcb7e0..e7f20eeb 100644 --- a/src/repoma/check_dev_files/__init__.py +++ b/src/repoma/check_dev_files/__init__.py @@ -2,9 +2,10 @@ from __future__ import annotations +import re import sys from argparse import ArgumentParser -from typing import Sequence +from typing import TYPE_CHECKING, Any, Sequence from repoma.check_dev_files.deprecated import remove_deprecated_tools from repoma.utilities.executor import Executor @@ -13,6 +14,7 @@ black, citation, commitlint, + conda, cspell, editorconfig, github_labels, @@ -26,6 +28,7 @@ pyright, pytest, pyupgrade, + readthedocs, release_drafter, ruff, setup_cfg, @@ -34,6 +37,9 @@ vscode, ) +if TYPE_CHECKING: + from repoma.utilities.project_info import PythonVersion + def main(argv: Sequence[str] | None = None) -> int: parser = _create_argparse() @@ -42,9 +48,11 @@ def main(argv: Sequence[str] | None = None) -> int: if not args.repo_title: args.repo_title = args.repo_name has_notebooks = not args.no_notebooks + dev_python_version = __get_python_version(args.dev_python_version) executor = Executor() executor(citation.main) executor(commitlint.main) + executor(conda.main, dev_python_version) executor(cspell.main, args.no_cspell_update) executor(editorconfig.main, args.no_python) if not args.allow_labels: @@ -54,6 +62,7 @@ def main(argv: Sequence[str] | None = None) -> int: github_workflows.main, allow_deprecated=args.allow_deprecated_workflows, doc_apt_packages=_to_list(args.doc_apt_packages), + python_version=dev_python_version, no_macos=args.no_macos, no_pypi=args.no_pypi, no_version_branches=args.no_version_branches, @@ -82,9 +91,10 @@ def main(argv: Sequence[str] | None = None) -> int: update_pip_constraints.main, cron_frequency=args.pin_requirements, ) + executor(readthedocs.main, dev_python_version) executor(remove_deprecated_tools, args.keep_issue_templates) executor(vscode.main, has_notebooks) - executor(gitpod.main, args.no_gitpod) + executor(gitpod.main, args.no_gitpod, dev_python_version) executor(precommit.main) return executor.finalize(exception=False) @@ -178,6 +188,13 @@ def _create_argparse() -> ArgumentParser: default=False, help="Do not perform the check on labels.toml", ) + parser.add_argument( + "--dev-python-version", + default="3.8", + help="Specify the Python version for your developer environment", + required=False, + type=str, + ) parser.add_argument( "--no-macos", action="store_true", @@ -263,5 +280,16 @@ def _to_list(arg: str) -> list[str]: return sorted(space_separated.split(" ")) +def __get_python_version(arg: Any) -> PythonVersion: + if not isinstance(arg, str): + msg = f"--dev-python-version must be a string, not {type(arg).__name__}" + raise TypeError(msg) + arg = arg.strip() + if not re.match(r"^3\.\d+$", arg): + msg = f"Invalid Python version: {arg}" + raise ValueError(msg) + return arg # type: ignore[return-value] + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/repoma/check_dev_files/conda.py b/src/repoma/check_dev_files/conda.py new file mode 100644 index 00000000..e5e10c71 --- /dev/null +++ b/src/repoma/check_dev_files/conda.py @@ -0,0 +1,73 @@ +"""Update the :file:`environment.yml` Conda environment file.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ruamel.yaml.scalarstring import PlainScalarString + +from repoma.errors import PrecommitError +from repoma.utilities import CONFIG_PATH +from repoma.utilities.project_info import PythonVersion, get_constraints_file +from repoma.utilities.yaml import create_prettier_round_trip_yaml + +if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + +def main(python_version: PythonVersion) -> None: + if not CONFIG_PATH.conda.exists(): + return + yaml = create_prettier_round_trip_yaml() + conda_env: CommentedMap = yaml.load(CONFIG_PATH.conda) + conda_deps: CommentedSeq = conda_env.get("dependencies", []) + + updated = _update_python_version(python_version, conda_deps) + updated |= _update_pip_dependencies(python_version, conda_deps) + if updated: + yaml.dump(conda_env, CONFIG_PATH.conda) + msg = f"Set the Python version in {CONFIG_PATH.conda} to {python_version}" + raise PrecommitError(msg) + + +def _update_python_version(version: PythonVersion, conda_deps: CommentedSeq) -> bool: + idx = __find_python_dependency_index(conda_deps) + expected = f"python=={version}.*" + if idx is not None and conda_deps[idx] != expected: + conda_deps[idx] = expected + return True + return False + + +def _update_pip_dependencies(version: PythonVersion, conda_deps: CommentedSeq) -> bool: + pip_deps = __get_pip_dependencies(conda_deps) + if pip_deps is None: + return False + constraints_file = get_constraints_file(version) + if constraints_file is None: + expected_pip = "-e .[dev]" + else: + expected_pip = f"-c {constraints_file} -e .[dev]" + if len(pip_deps) and pip_deps[0] != expected_pip: + pip_deps[0] = PlainScalarString(expected_pip) + return True + return False + + +def __find_python_dependency_index(dependencies: CommentedSeq) -> int | None: + for i, dep in enumerate(dependencies): + if not isinstance(dep, str): + continue + if dep.strip().startswith("python"): + return i + return None + + +def __get_pip_dependencies(dependencies: CommentedSeq) -> CommentedSeq | None: + for dep in dependencies: + if not isinstance(dep, dict): + continue + pip_deps = dep.get("pip") + if pip_deps is not None and isinstance(pip_deps, list): + return pip_deps + return None diff --git a/src/repoma/check_dev_files/github_workflows.py b/src/repoma/check_dev_files/github_workflows.py index 0ad61b61..de8dfb66 100644 --- a/src/repoma/check_dev_files/github_workflows.py +++ b/src/repoma/check_dev_files/github_workflows.py @@ -13,7 +13,7 @@ from repoma.utilities import CONFIG_PATH, REPOMA_DIR, hash_file, write from repoma.utilities.executor import Executor from repoma.utilities.precommit import PrecommitConfig -from repoma.utilities.project_info import get_pypi_name +from repoma.utilities.project_info import PythonVersion, get_pypi_name from repoma.utilities.vscode import ( add_extension_recommendation, remove_extension_recommendation, @@ -34,6 +34,7 @@ def main( no_macos: bool, no_pypi: bool, no_version_branches: bool, + python_version: PythonVersion, single_threaded: bool, skip_tests: list[str], test_extras: list[str], @@ -45,6 +46,7 @@ def main( allow_deprecated, doc_apt_packages, no_macos, + python_version, single_threaded, skip_tests, test_extras, @@ -59,7 +61,7 @@ def update() -> None: yaml = create_prettier_round_trip_yaml() workflow_path = CONFIG_PATH.github_workflow_dir / "cd.yml" expected_data = yaml.load(REPOMA_DIR / workflow_path) - if no_pypi or not os.path.exists(CONFIG_PATH.setup_cfg): + if no_pypi or not CONFIG_PATH.setup_cfg.exists(): del expected_data["jobs"]["pypi"] if no_version_branches: del expected_data["jobs"]["push"] @@ -93,6 +95,7 @@ def _update_ci_workflow( allow_deprecated: bool, doc_apt_packages: list[str], no_macos: bool, + python_version: PythonVersion, single_threaded: bool, skip_tests: list[str], test_extras: list[str], @@ -102,6 +105,7 @@ def update() -> None: REPOMA_DIR / CONFIG_PATH.github_workflow_dir / "ci.yml", doc_apt_packages, no_macos, + python_version, single_threaded, skip_tests, test_extras, @@ -135,32 +139,41 @@ def _get_ci_workflow( path: Path, doc_apt_packages: list[str], no_macos: bool, + python_version: PythonVersion, single_threaded: bool, skip_tests: list[str], test_extras: list[str], ) -> tuple[YAML, dict]: yaml = create_prettier_round_trip_yaml() config = yaml.load(path) - __update_doc_section(config, doc_apt_packages) + __update_doc_section(config, doc_apt_packages, python_version) __update_pytest_section(config, no_macos, single_threaded, skip_tests, test_extras) - __update_style_section(config) + __update_style_section(config, python_version) return yaml, config -def __update_doc_section(config: CommentedMap, apt_packages: list[str]) -> None: +def __update_doc_section( + config: CommentedMap, apt_packages: list[str], python_version: PythonVersion +) -> None: if not os.path.exists("docs/"): del config["jobs"]["doc"] else: with_section = config["jobs"]["doc"]["with"] + if python_version != "3.8": + with_section["python-version"] = DoubleQuotedScalarString(python_version) if apt_packages: with_section["apt-packages"] = " ".join(apt_packages) - if not os.path.exists(CONFIG_PATH.readthedocs): + if not CONFIG_PATH.readthedocs.exists(): with_section["gh-pages"] = True __update_with_section(config, job_name="doc") -def __update_style_section(config: CommentedMap) -> None: - if not os.path.exists(CONFIG_PATH.precommit): +def __update_style_section(config: CommentedMap, python_version: PythonVersion) -> None: + if python_version != "3.8": + config["jobs"]["style"]["with"] = { + "python-version": DoubleQuotedScalarString(python_version) + } + if not CONFIG_PATH.precommit.exists(): del config["jobs"]["style"] else: cfg = PrecommitConfig.load() @@ -182,7 +195,7 @@ def __update_pytest_section( with_section = config["jobs"]["pytest"]["with"] if test_extras: with_section["additional-extras"] = ",".join(test_extras) - if os.path.exists(CONFIG_PATH.codecov): + if CONFIG_PATH.codecov.exists(): with_section["coverage-target"] = __get_package_name() if not no_macos: with_section["macos-python-version"] = DoubleQuotedScalarString("3.9") diff --git a/src/repoma/check_dev_files/gitpod.py b/src/repoma/check_dev_files/gitpod.py index 0681567b..2f2fdef3 100644 --- a/src/repoma/check_dev_files/gitpod.py +++ b/src/repoma/check_dev_files/gitpod.py @@ -7,23 +7,24 @@ from repoma.errors import PrecommitError from repoma.utilities import CONFIG_PATH, REPOMA_DIR -from repoma.utilities.project_info import get_repo_url +from repoma.utilities.project_info import ( + PythonVersion, + get_constraints_file, + get_repo_url, +) from repoma.utilities.readme import add_badge from repoma.utilities.yaml import write_yaml -__CONSTRAINTS_FILE = ".constraints/py3.8.txt" - -def main(no_gitpod: bool) -> None: +def main(no_gitpod: bool, python_version: PythonVersion) -> None: if no_gitpod: if CONFIG_PATH.gitpod.exists(): os.remove(CONFIG_PATH.gitpod) msg = f"Removed {CONFIG_PATH.gitpod} as requested by --no-gitpod" raise PrecommitError(msg) return - pin_dependencies = os.path.exists(__CONSTRAINTS_FILE) error_message = "" - expected_config = _generate_gitpod_config(pin_dependencies) + expected_config = _generate_gitpod_config(python_version) if CONFIG_PATH.gitpod.exists(): with open(CONFIG_PATH.gitpod) as stream: existing_config = yaml.load(stream, Loader=yaml.SafeLoader) @@ -51,14 +52,16 @@ def _extract_extensions() -> dict: return {} -def _generate_gitpod_config(pin_dependencies: bool) -> dict: +def _generate_gitpod_config(python_version: PythonVersion) -> dict: with open(REPOMA_DIR / ".template" / CONFIG_PATH.gitpod) as stream: gitpod_config = yaml.load(stream, Loader=yaml.SafeLoader) tasks = gitpod_config["tasks"] - if pin_dependencies: - tasks[0]["init"] = f"pip install -c {__CONSTRAINTS_FILE} -e .[dev]" + tasks[0]["init"] = f"pyenv local {python_version}" + constraints_file = get_constraints_file(python_version) + if constraints_file is None: + tasks[1]["init"] = "pip install -e .[dev]" else: - tasks[0]["init"] = "pip install -e .[dev]" + tasks[1]["init"] = f"pip install -c {constraints_file} -e .[dev]" extensions = _extract_extensions() if extensions: gitpod_config["vscode"] = {"extensions": extensions} diff --git a/src/repoma/check_dev_files/readthedocs.py b/src/repoma/check_dev_files/readthedocs.py new file mode 100644 index 00000000..d126f4e0 --- /dev/null +++ b/src/repoma/check_dev_files/readthedocs.py @@ -0,0 +1,81 @@ +"""Update Read the Docs configuration.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ruamel.yaml.scalarstring import DoubleQuotedScalarString + +from repoma.errors import PrecommitError +from repoma.utilities import CONFIG_PATH +from repoma.utilities.executor import Executor +from repoma.utilities.project_info import PythonVersion, get_constraints_file +from repoma.utilities.yaml import create_prettier_round_trip_yaml + +if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + +def main(python_version: PythonVersion) -> None: + if not CONFIG_PATH.readthedocs.exists(): + return + executor = Executor() + executor(_update_os) + executor(_update_python_version, python_version) + executor(_update_install_step, python_version) + executor.finalize() + + +def _update_os() -> None: + yaml = create_prettier_round_trip_yaml() + config: CommentedMap = yaml.load(CONFIG_PATH.readthedocs) + build: CommentedMap | None = config.get("build") + if build is None: + return + os: str | None = build.get("os") + expected = "ubuntu-22.04" + if os == expected: + return + build["os"] = expected + yaml.dump(config, CONFIG_PATH.readthedocs) + msg = f"Switched to {expected} in {CONFIG_PATH.readthedocs}" + raise PrecommitError(msg) + + +def _update_python_version(python_version: PythonVersion) -> None: + yaml = create_prettier_round_trip_yaml() + config: CommentedMap = yaml.load(CONFIG_PATH.readthedocs) + tools: CommentedMap = config.get("build", {}).get("tools", {}) + if tools is None: + return + existing_version = tools.get("python") + if existing_version is None: + return + expected_version = DoubleQuotedScalarString(python_version) + if expected_version == existing_version: + return + tools["python"] = expected_version + yaml.dump(config, CONFIG_PATH.readthedocs) + msg = f"Switched to Python {python_version} in {CONFIG_PATH.readthedocs}" + raise PrecommitError(msg) + + +def _update_install_step(python_version: PythonVersion) -> None: + yaml = create_prettier_round_trip_yaml() + config: CommentedMap = yaml.load(CONFIG_PATH.readthedocs) + steps: CommentedSeq = config.get("build", {}).get("jobs", {}).get("post_install") + if steps is None: + return + if len(steps) == 0: + return + constraints_file = get_constraints_file(python_version) + if constraints_file is None: + expected_install = "pip install -e .[doc]" + else: + expected_install = f"pip install -c {constraints_file} -e .[doc]" + if steps[0] == expected_install: + return + steps[0] = expected_install + yaml.dump(config, CONFIG_PATH.readthedocs) + msg = f"Pinned constraints for Python {python_version} in {CONFIG_PATH.readthedocs}" + raise PrecommitError(msg) diff --git a/src/repoma/utilities/__init__.py b/src/repoma/utilities/__init__.py index bbbb3031..cfafac28 100644 --- a/src/repoma/utilities/__init__.py +++ b/src/repoma/utilities/__init__.py @@ -19,6 +19,7 @@ class _ConfigFilePaths(NamedTuple): citation: Path = Path("CITATION.cff") codecov: Path = Path("codecov.yml") + conda: Path = Path("environment.yml") cspell: Path = Path(".cspell.json") editorconfig: Path = Path(".editorconfig") github_workflow_dir: Path = Path(".github/workflows") diff --git a/src/repoma/utilities/project_info.py b/src/repoma/utilities/project_info.py index 5e4e37fe..d85bf4f0 100644 --- a/src/repoma/utilities/project_info.py +++ b/src/repoma/utilities/project_info.py @@ -4,6 +4,7 @@ import os import sys +from pathlib import Path from textwrap import dedent from typing import TYPE_CHECKING @@ -159,3 +160,10 @@ def open_setup_cfg() -> ConfigParser: msg = "This repository contains no setup.cfg file" raise PrecommitError(msg) return open_config(CONFIG_PATH.setup_cfg) + + +def get_constraints_file(python_version: PythonVersion) -> Path | None: + path = Path(f".constraints/py{python_version}.txt") + if path.exists(): + return path + return None diff --git a/tests/check_dev_files/test_gitpod.py b/tests/check_dev_files/test_gitpod.py index 52ddd6a1..503e7ac9 100644 --- a/tests/check_dev_files/test_gitpod.py +++ b/tests/check_dev_files/test_gitpod.py @@ -1,11 +1,8 @@ -import pytest - from repoma.check_dev_files.gitpod import _extract_extensions, _generate_gitpod_config -@pytest.mark.parametrize("pin_dependencies", [False, True]) -def test_get_gitpod_content(pin_dependencies: bool): - gitpod_content = _generate_gitpod_config(pin_dependencies) +def test_get_gitpod_content(): + gitpod_content = _generate_gitpod_config("3.8") assert set(gitpod_content) == { "github", "tasks", @@ -22,12 +19,8 @@ def test_get_gitpod_content(pin_dependencies: bool): "pullRequestsFromForks": True, } } - if pin_dependencies: - assert gitpod_content["tasks"] == [ - {"init": "pip install -c .constraints/py3.8.txt -e .[dev]"}, - ] - else: - assert gitpod_content["tasks"] == [ - {"init": "pip install -e .[dev]"}, - ] + assert gitpod_content["tasks"] == [ + {"init": "pyenv local 3.8"}, + {"init": "pip install -e .[dev]"}, + ] assert gitpod_content["vscode"]["extensions"] == _extract_extensions()