diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4cf530d..0879ee0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,11 +9,16 @@ on: - "renovate/*" pull_request: +# Cancel currently-running builds when pushing new commits to a PR or branch +# https://stackoverflow.com/a/72408109 +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Check out code uses: actions/checkout@v4 with: fetch-depth: 0 @@ -42,7 +47,7 @@ jobs: echo "::group::Wait for snap to complete" snap watch --last=install echo "::endgroup::" - - name: Run Linters + - name: Run linters run: tox run --skip-pkg-install --no-list-dependencies --colored yes -m lint unit: strategy: @@ -50,7 +55,8 @@ jobs: platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest] runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4 + - name: Check out code + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python @@ -90,6 +96,7 @@ jobs: platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest] runs-on: ${{ matrix.platform }} steps: + - name: Check out code - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.gitignore b/.gitignore index 289a38a..0aaac19 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ *.egg-info/ .installed.cfg *.egg +*.snap MANIFEST # PyInstaller @@ -71,6 +72,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/spelling/ # PyBuilder target/ @@ -103,6 +105,7 @@ celerybeat.pid *.sage.py # Environments +.direnv .env .venv env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd6d355..50e6d3d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,12 +15,11 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.6.9" hooks: + # Run the linter - id: ruff args: [--fix, --exit-non-zero-on-fix] - - repo: https://github.com/psf/black - rev: "24.8.0" - hooks: - - id: black + # Run the formatter + - id: ruff-format - repo: https://github.com/adrienverge/yamllint.git rev: "v1.35.1" hooks: diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..e69de29 diff --git a/Makefile b/Makefile index 6dcafe4..01eb849 100644 --- a/Makefile +++ b/Makefile @@ -6,19 +6,44 @@ ifneq ($(OS),Windows_NT) OS := $(shell uname) endif +.DEFAULT_GOAL := help + +.ONESHELL: + +.SHELLFLAGS = -ec + .PHONY: help help: ## Show this help. - @printf "%-30s %s\n" "Target" "Description" - @printf "%-30s %s\n" "------" "-----------" - @fgrep " ## " $(MAKEFILE_LIST) | fgrep -v grep | awk -F ': .*## ' '{$$1 = sprintf("%-30s", $$1)} 1' + @printf "%-41s %s\n" "Target" "Description" + @printf "%-41s %s\n" "------" "-----------" + @fgrep " ##" $(MAKEFILE_LIST) | fgrep -v grep | sed 's/:[^#]*/ /' | awk -F '[: ]*' \ + '{ + if ($$2 == "##") + { + $$1=sprintf("%-40s", $$1); + $$2=""; + print $$0; + } + else + { + $$1=sprintf(" └%-38s", $$1); + $$2=""; + print $$0; + } + }' + +---------------- : ## ---------------- .PHONY: setup setup: ## Set up a development environment ifeq ($(OS),Linux) - sudo snap install codespell ruff shellcheck - sudo snap install --classic --beta astral-uv - sudo snap alias astral-uv.uv uv - sudo snap alias astral-uv.uvx uvx + changes="`sudo snap install --no-wait codespell`" + changes="${changes} `sudo snap install --no-wait ruff`" + changes="${changes} `sudo snap install --no-wait shellcheck`" + changes="${changes} `sudo snap install --classic --no-wait astral-uv`" + for change in ${changes}; do + snap watch ${change} + done else ifeq ($(OS),Windows_NT) pipx install uv choco install shellcheck @@ -39,68 +64,87 @@ ifeq (, $(shell which pre-commit)) endif pre-commit install +---------------- : ## ---------------- + .PHONY: autoformat autoformat: format-ruff format-codespell ## Run all automatic formatters -.PHONY: lint -lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs ## Run all linters - -.PHONY: test -test: test-unit test-integration ## Run all tests - -.PHONY: docs -docs: ## Build documentation - uv run --extra docs sphinx-build -b html -W docs docs/_build +.PHONY: format-ruff +format-ruff: ##- Automatically format with ruff + ruff check --fix $(SOURCES) + success=true + ruff check --fix $(SOURCES) || success=false + ruff format $(SOURCES) + $success || exit 1 -.PHONY: docs-auto -docs-auto: ## Build and host docs with sphinx-autobuild - uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build +.PHONY: format-codespell +format-codespell: ##- Fix spelling issues with codespell + uv run codespell --toml pyproject.toml --write-changes $(SOURCES) -# Helpful in `help` to split the main targets from things that build ---------------- : ## ---------------- -.PHONY: format-codespell -format-codespell: ## Fix spelling issues with codespell - uv run codespell --toml pyproject.toml --write-changes $(SOURCES) +.PHONY: lint +lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs ## Run all linters -.PHONY: format-ruff -format-ruff: ## Automatically format with ruff - ruff format $(SOURCES) - ruff check --fix $(SOURCES) +.PHONY: lint-ruff +lint-ruff: ##- Lint with ruff + ruff check $(SOURCES) + ruff format --diff $(SOURCES) .PHONY: lint-codespell -lint-codespell: ## Check spelling with codespell +lint-codespell: ##- Check spelling with codespell uv run codespell --toml pyproject.toml $(SOURCES) -.PHONY: lint-docs -lint-docs: ## Lint the documentation - uv run --extra docs sphinx-lint --max-line-length 80 --enable all $(DOCS) - .PHONY: lint-mypy -lint-mypy: ## Check types with mypy - uv run mypy $(SOURCES) +lint-mypy: ##- Check types with mypy + uv run mypy --show-traceback --show-error-codes $(SOURCES) .PHONY: lint-pyright -lint-pyright: ## Check types with pyright +lint-pyright: ##- Check types with pyright + # Fix for a bug in npm + [ -d "/home/ubuntu/.npm/_cacache" ] && chown -R 1000:1000 "/home/ubuntu/.npm" || true uv run pyright -.PHONY: lint-ruff -lint-ruff: ## Lint with ruff - ruff format --diff $(SOURCES) - ruff check $(SOURCES) - .PHONY: lint-shellcheck -lint-shellcheck: - sh -c 'git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck' +lint-shellcheck: ##- Lint shell scripts + git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck .PHONY: lint-yaml -lint-yaml: ## Lint YAML files with yamllint +lint-yaml: ##- Lint YAML files with yamllint uv run yamllint . +.PHONY: lint-docs +lint-docs: ##- Lint the documentation + uv run --extra docs sphinx-lint --max-line-length 88 --enable all $(DOCS) + +---------------- : ## ---------------- + +.PHONY: test +test: test-unit test-integration ## Run all tests + .PHONY: test-unit -test-unit: ## Run unit tests +test-unit: ##- Run unit tests uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.unit.xml --junit-xml=.results.unit.xml tests/unit .PHONY: test-integration -test-integration: ## Run integration tests +test-integration: ##- Run integration tests uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.integration.xml --junit-xml=.results.integration.xml tests/integration + +---------------- : ## ---------------- + +.PHONY: coverage +coverage: ## Generate coverage report + coverage run --source starcraft -m pytest + coverage xml -o coverage.xml + coverage report -m + coverage html + +---------------- : ## ---------------- + +.PHONY: docs +docs: ## Build documentation + uv run --extra docs sphinx-build -b html -W docs docs/_build + +.PHONY: docs-auto +docs-auto: ## Build and host docs with sphinx-autobuild + uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build diff --git a/README.rst b/README.rst index 0f173e4..bd323ab 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,64 @@ TODO Migrate existing projects -------------------------------- -TODO +#. Update this guide as you go along, if something is unclear or missing. +#. Use ruff. + #. Pull in the bare minimum ``pyproject.toml`` needed to use ruff. + #. Make your codebase pass with ruff. Commit after each step: + #. ``ruff check --fix`` + #. ``ruff check --fix --unsafe-fixes`` + #. ``ruff check --add-noqa`` + #. ``ruff format`` + #. Replace use of black, flake8, pydocstyle, isort, and pylint in Makefile/CI + with: + - ``ruff check --fix`` + - ``ruff format`` +#. Modify top-level files in your project to match what's in Starbase as closely + as possible. + #. ``Makefile`` - Ensure you use ``uv`` and at least have the same targets: + - ``setup`` + - ``lint`` + - ``test-unit`` + - ``test-integration`` (If this applies to your repo, i.e. the repo is a library + rather than an application) + - ``coverage`` + #. ``pyproject.toml`` - Expand from just the ruff things: move things into + here from your ``setup.py``, ``setup.cfg``, and ``requirements.*.txt``. + #. ``README`` - If your readme is .md, convert to .rst with pandoc: + ``pandoc -o README.rst README.md`` + Don't worry about making the contents match, Starbase's is very specific. +#. Run all the linters: ``make lint`` + #. ``mypy``: + - Mypy checks the same things as ``ruff``'s ``ANNXXX`` checks, but + ``ruff``'s ``noqa`` directives mean nothing to mypy. You'll need to fix + these by hand, mostly by adding type annotations to function definitions. + #. ``pyright``: + - For errors along the lines of "Stub file not found for $library", check + for the existence of pip package ``typing-$library`` and add it as a + dependency. + - If you have lots of errors you may need to remove the ``strict`` + directive from ``pyproject.toml``. +#. Do a side-by-side diff of the ``.gitignore`` files in your project and + Starbase, making them as close as possible and adding anything that makes + sense upstream. +#. Bring in remaining top-level files: + - .editorconfig + - .pre-commit-config.yaml + - .shellcheckrc + - tox.ini + - .yamllint.yaml +#. If you're rebasing a library, add the integrations tests structure. + Applications should use spread for integration tests. +# Finally, once all files are manually synced, actually sync the git history: + - ``git remote add starbase git@github.com:canonical/starbase.git`` + - ``git merge --allow-unrelated-histories starbase/main`` + - ``git remote remove starbase`` + - Don't forget to review all the new files and dirs that this merge adds - + you'll want to delete a lot of them. + - When you merge, DO NOT squash, otherwise the starbase history will not be + preserved. + + Create a new project --------------------------- diff --git a/pyproject.toml b/pyproject.toml index d76a0c2..29a4b9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [project] name = "starcraft" +description = "The basis for Starcraft team packages" dynamic = ["version", "readme"] dependencies = [ @@ -157,10 +158,7 @@ strict = false line-length = 88 target-version = "py310" src = ["starcraft", "tests"] -extend-exclude = [ - "docs", - "__pycache__", -] +extend-exclude = ["docs"] [tool.ruff.format] docstring-code-format = true @@ -169,6 +167,7 @@ quote-style = "double" [tool.ruff.lint] # Follow ST063 - Maintaining and updating linting specifications for updating these. +# Handy link: https://docs.astral.sh/ruff/rules/ select = [ # Base linting rule selections. # See the internal document for discussion: # https://docs.google.com/document/d/1i1n8pDmFmWi4wTDpk-JfnWCVUThPJiggyPi2DYwBBu4/edit @@ -285,7 +284,7 @@ max-args = 8 classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"] [tool.ruff.lint.per-file-ignores] -"tests/**.py" = [ # Some things we want for the moin project are unnecessary in tests. +"tests/**.py" = [ # Some things we want for the main project are unnecessary in tests. "D", # Ignore docstring rules in tests "ANN", # Ignore type annotations in tests "ARG", # Allow unused arguments in tests (e.g. for fake functions/methods/classes) diff --git a/tox.ini b/tox.ini index b101421..dd4e619 100644 --- a/tox.ini +++ b/tox.ini @@ -78,8 +78,8 @@ labels = lint commands_pre = shellcheck: bash -c '{[shellcheck]find} | {[shellcheck]filter} > {env_tmp_dir}/shellcheck_files' commands = - ruff: ruff format --check --diff --respect-gitignore {posargs:.} - ruff: ruff check --respect-gitignore {posargs:.} + ruff: ruff format --check --diff {posargs:.} + ruff: ruff check {posargs:.} shellcheck: xargs -ra {env_tmp_dir}/shellcheck_files shellcheck codespell: codespell --toml {tox_root}/pyproject.toml {posargs} yaml: yamllint {posargs} .