feat: upstream the good parts when rebasing application (#265)

* feat: upstream good parts from Makefile/pyproject * feat(docs): add start of migration instructions * fix: remove unused variable * fix: need to set variable to keep uv from complaining * feat: python-apt launchpad dep in pyproject.toml * feat: more upstreaming - Compared .gitignores and added things from my application. - Removed redundant black things in hook file. - Ruff format should be after check, as sometimes check's output needs to be formatted, but format's output shouldn't need to be checked. - Updated refactor instructions. - Updated ruff excludes. - Helped pyright's type checking. * Apply suggestions from code review Co-authored-by: Alex Lowe <alex.lowe@canonical.com> * fix: Alex's PR review * feat(Makefile): upstream coverage target * feat(docs): update readme with integration test info * feat: upstream minor tests workflow things * fix: uv snap has added aliases and doesn't need beta * docs: finishing touches on rebase instructions * fix: the uv.lock file is supposed to be checked in * feat(docs): make targets called out per @lengau, add update note * Apply suggestions from code review * fix(Makefile): upstream npm pyright bugfix * fix: add in the ruff formatter, per @dariuszd21 * fix: don't shellcheck everything that gets sourced * Update Makefile Co-authored-by: Dariusz Duda <darekduda21@gmail.com> * feat(docs): add note about merging * feat: allow installing snaps in parallel Co-authored-by: Alex Lower <alex.lowe@canonical.com> * Update Makefile * fix: Darius pointed out that ruff excludes via .gitignore * fix: `--respect-gitignore` is the default --------- Co-authored-by: Alex Lowe <alex.lowe@canonical.com> Co-authored-by: Dariusz Duda <darekduda21@gmail.com>
canonical · Oct 11, 2024 · 44ce081 · 44ce081
1 parent 84c2a0e
commit 44ce081
Show file tree

Hide file tree

Showing 8 changed files with 169 additions and 60 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -9,11 +9,16 @@ on:
       - "renovate/*"
   pull_request:
 
+# Cancel currently-running builds when pushing new commits to a PR or branch
+# https://stackoverflow.com/a/72408109
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
+      - name: Check out code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -42,15 +47,16 @@ jobs:
           echo "::group::Wait for snap to complete"
           snap watch --last=install
           echo "::endgroup::"
-      - name: Run Linters
+      - name: Run linters
         run: tox run --skip-pkg-install --no-list-dependencies --colored yes -m lint
   unit:
     strategy:
       matrix:
         platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out code
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
@@ -90,6 +96,7 @@ jobs:
         platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest]
     runs-on: ${{ matrix.platform }}
     steps:
+      - name: Check out code
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0

diff --git a/.gitignore b/.gitignore
@@ -25,6 +25,7 @@ share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+*.snap
 MANIFEST
 
 # PyInstaller
@@ -71,6 +72,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/spelling/
 
 # PyBuilder
 target/
@@ -103,6 +105,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.direnv
 .env
 .venv
 env/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,12 +15,11 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.6.9"
     hooks:
+      # Run the linter
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-  - repo: https://github.com/psf/black
-    rev: "24.8.0"
-    hooks:
-      - id: black
+      # Run the formatter
+      - id: ruff-format
   - repo: https://github.com/adrienverge/yamllint.git
     rev: "v1.35.1"
     hooks:

diff --git a/.shellcheckrc b/.shellcheckrc
diff --git a/Makefile b/Makefile
@@ -6,19 +6,44 @@ ifneq ($(OS),Windows_NT)
 	OS := $(shell uname)
 endif
 
+.DEFAULT_GOAL := help
+
+.ONESHELL:
+
+.SHELLFLAGS = -ec
+
 .PHONY: help
 help: ## Show this help.
-	@printf "%-30s %s\n" "Target" "Description"
-	@printf "%-30s %s\n" "------" "-----------"
-	@fgrep " ## " $(MAKEFILE_LIST) | fgrep -v grep | awk -F ': .*## ' '{$$1 = sprintf("%-30s", $$1)} 1'
+	@printf "%-41s %s\n" "Target" "Description"
+	@printf "%-41s %s\n" "------" "-----------"
+	@fgrep " ##" $(MAKEFILE_LIST) | fgrep -v grep | sed 's/:[^#]*/ /' | awk -F '[: ]*' \
+	'{
+		if ($$2 == "##")
+		{
+			$$1=sprintf("%-40s", $$1);
+			$$2="";
+			print $$0;
+		}
+		else
+		{
+			$$1=sprintf(" └%-38s", $$1);
+			$$2="";
+			print $$0;
+		}
+	}'
+
+---------------- : ## ----------------
 
 .PHONY: setup
 setup: ## Set up a development environment
 ifeq ($(OS),Linux)
-	sudo snap install codespell ruff shellcheck
-	sudo snap install --classic --beta astral-uv
-	sudo snap alias astral-uv.uv uv
-	sudo snap alias astral-uv.uvx uvx
+	changes="`sudo snap install --no-wait codespell`"
+	changes="${changes} `sudo snap install --no-wait ruff`"
+	changes="${changes} `sudo snap install --no-wait shellcheck`"
+	changes="${changes} `sudo snap install --classic --no-wait astral-uv`"
+	for change in ${changes}; do
+		snap watch ${change}
+	done
 else ifeq ($(OS),Windows_NT)
 	pipx install uv
 	choco install shellcheck
@@ -39,68 +64,87 @@ ifeq (, $(shell which pre-commit))
 endif
 	pre-commit install
 
+---------------- : ## ----------------
+
 .PHONY: autoformat
 autoformat: format-ruff format-codespell  ## Run all automatic formatters
 
-.PHONY: lint
-lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs  ## Run all linters
-
-.PHONY: test
-test: test-unit test-integration  ## Run all tests
-
-.PHONY: docs
-docs: ## Build documentation
-	uv run --extra docs sphinx-build -b html -W docs docs/_build
+.PHONY: format-ruff
+format-ruff:  ##- Automatically format with ruff
+	ruff check --fix $(SOURCES)
+	success=true
+	ruff check --fix $(SOURCES) || success=false
+	ruff format $(SOURCES)
+	$success || exit 1
 
-.PHONY: docs-auto
-docs-auto:  ## Build and host docs with sphinx-autobuild
-	uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build
+.PHONY: format-codespell
+format-codespell:  ##- Fix spelling issues with codespell
+	uv run codespell --toml pyproject.toml --write-changes $(SOURCES)
 
-# Helpful in `help` to split the main targets from things that build
 ---------------- : ## ----------------
 
-.PHONY: format-codespell
-format-codespell:  ## Fix spelling issues with codespell
-	uv run codespell --toml pyproject.toml --write-changes $(SOURCES)
+.PHONY: lint
+lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs  ## Run all linters
 
-.PHONY: format-ruff
-format-ruff:  ## Automatically format with ruff
-	ruff format $(SOURCES)
-	ruff check --fix $(SOURCES)
+.PHONY: lint-ruff
+lint-ruff:  ##- Lint with ruff
+	ruff check $(SOURCES)
+	ruff format --diff $(SOURCES)
 
 .PHONY: lint-codespell
-lint-codespell: ## Check spelling with codespell
+lint-codespell:  ##- Check spelling with codespell
 	uv run codespell --toml pyproject.toml $(SOURCES)
 
-.PHONY: lint-docs
-lint-docs:  ## Lint the documentation
-	uv run --extra docs sphinx-lint --max-line-length 80 --enable all $(DOCS)
-
 .PHONY: lint-mypy
-lint-mypy: ## Check types with mypy
-	uv run mypy $(SOURCES)
+lint-mypy:  ##- Check types with mypy
+	uv run mypy --show-traceback --show-error-codes $(SOURCES)
 
 .PHONY: lint-pyright
-lint-pyright: ## Check types with pyright
+lint-pyright:  ##- Check types with pyright
+	# Fix for a bug in npm
+	[ -d "/home/ubuntu/.npm/_cacache" ] && chown -R 1000:1000 "/home/ubuntu/.npm" || true
 	uv run pyright
 
-.PHONY: lint-ruff
-lint-ruff:  ## Lint with ruff
-	ruff format --diff $(SOURCES)
-	ruff check $(SOURCES)
-
 .PHONY: lint-shellcheck
-lint-shellcheck:
-	sh -c 'git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck'
+lint-shellcheck:  ##- Lint shell scripts
+	git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck
 
 .PHONY: lint-yaml
-lint-yaml:  ## Lint YAML files with yamllint
+lint-yaml:  ##- Lint YAML files with yamllint
 	uv run yamllint .
 
+.PHONY: lint-docs
+lint-docs:  ##- Lint the documentation
+	uv run --extra docs sphinx-lint --max-line-length 88 --enable all $(DOCS)
+
+---------------- : ## ----------------
+
+.PHONY: test
+test: test-unit test-integration  ## Run all tests
+
 .PHONY: test-unit
-test-unit: ## Run unit tests
+test-unit:  ##- Run unit tests
 	uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.unit.xml --junit-xml=.results.unit.xml tests/unit
 
 .PHONY: test-integration
-test-integration:  ## Run integration tests
+test-integration:  ##- Run integration tests
 	uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.integration.xml --junit-xml=.results.integration.xml tests/integration
+
+---------------- : ## ----------------
+
+.PHONY: coverage
+coverage:  ## Generate coverage report
+	coverage run --source starcraft -m pytest
+	coverage xml -o coverage.xml
+	coverage report -m
+	coverage html
+
+---------------- : ## ----------------
+
+.PHONY: docs
+docs:  ## Build documentation
+	uv run --extra docs sphinx-build -b html -W docs docs/_build
+
+.PHONY: docs-auto
+docs-auto:  ## Build and host docs with sphinx-autobuild
+	uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build
diff --git a/README.rst b/README.rst
@@ -25,7 +25,64 @@ TODO
 
 Migrate existing projects
 --------------------------------
-TODO
+#. Update this guide as you go along, if something is unclear or missing.
+#. Use ruff.
+   #. Pull in the bare minimum ``pyproject.toml`` needed to use ruff.
+   #. Make your codebase pass with ruff.  Commit after each step:
+      #. ``ruff check --fix``
+      #. ``ruff check --fix --unsafe-fixes``
+      #. ``ruff check --add-noqa``
+      #. ``ruff format``
+   #. Replace use of black, flake8, pydocstyle, isort, and pylint in Makefile/CI
+      with:
+      - ``ruff check --fix``
+      - ``ruff format``
+#. Modify top-level files in your project to match what's in Starbase as closely
+   as possible.
+   #. ``Makefile`` - Ensure you use ``uv`` and at least have the same targets:
+      - ``setup``
+      - ``lint``
+      - ``test-unit``
+      - ``test-integration`` (If this applies to your repo, i.e. the repo is a library
+        rather than an application)
+      - ``coverage``
+   #. ``pyproject.toml`` - Expand from just the ruff things: move things into
+      here from your ``setup.py``, ``setup.cfg``, and ``requirements.*.txt``.
+   #. ``README`` - If your readme is .md, convert to .rst with pandoc:
+      ``pandoc -o README.rst README.md``
+      Don't worry about making the contents match, Starbase's is very specific.
+#. Run all the linters: ``make lint``
+   #. ``mypy``:
+      - Mypy checks the same things as ``ruff``'s ``ANNXXX`` checks, but
+        ``ruff``'s ``noqa`` directives mean nothing to mypy.  You'll need to fix
+        these by hand, mostly by adding type annotations to function definitions.
+   #. ``pyright``:
+      - For errors along the lines of "Stub file not found for $library", check
+        for the existence of pip package ``typing-$library`` and add it as a
+        dependency.
+      - If you have lots of errors you may need to remove the ``strict``
+        directive from ``pyproject.toml``.
+#. Do a side-by-side diff of the ``.gitignore`` files in your project and
+   Starbase, making them as close as possible and adding anything that makes
+   sense upstream.
+#. Bring in remaining top-level files:
+   - .editorconfig
+   - .pre-commit-config.yaml
+   - .shellcheckrc
+   - tox.ini
+   - .yamllint.yaml
+#. If you're rebasing a library, add the integrations tests structure.
+   Applications should use spread for integration tests.
+# Finally, once all files are manually synced, actually sync the git history:
+   - ``git remote add starbase git@github.com:canonical/starbase.git``
+   - ``git merge --allow-unrelated-histories starbase/main``
+   - ``git remote remove starbase``
+   - Don't forget to review all the new files and dirs that this merge adds -
+     you'll want to delete a lot of them.
+   - When you merge, DO NOT squash, otherwise the starbase history will not be
+     preserved.
+
+
 
 Create a new project
 ---------------------------

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,6 @@
 [project]
 name = "starcraft"
+description = "The basis for Starcraft team packages"
 dynamic = ["version", "readme"]
 dependencies = [
 
@@ -157,10 +158,7 @@ strict = false
 line-length = 88
 target-version = "py310"
 src = ["starcraft", "tests"]
-extend-exclude = [
-    "docs",
-    "__pycache__",
-]
+extend-exclude = ["docs"]
 
 [tool.ruff.format]
 docstring-code-format = true
@@ -169,6 +167,7 @@ quote-style = "double"
 
 [tool.ruff.lint]
 # Follow ST063 - Maintaining and updating linting specifications for updating these.
+# Handy link: https://docs.astral.sh/ruff/rules/
 select = [  # Base linting rule selections.
     # See the internal document for discussion:
     # https://docs.google.com/document/d/1i1n8pDmFmWi4wTDpk-JfnWCVUThPJiggyPi2DYwBBu4/edit
@@ -285,7 +284,7 @@ max-args = 8
 classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/**.py" = [  # Some things we want for the moin project are unnecessary in tests.
+"tests/**.py" = [  # Some things we want for the main project are unnecessary in tests.
     "D",  # Ignore docstring rules in tests
     "ANN",  # Ignore type annotations in tests
     "ARG",  # Allow unused arguments in tests (e.g. for fake functions/methods/classes)

diff --git a/tox.ini b/tox.ini
@@ -78,8 +78,8 @@ labels = lint
 commands_pre =
     shellcheck: bash -c '{[shellcheck]find} | {[shellcheck]filter} > {env_tmp_dir}/shellcheck_files'
 commands =
-    ruff: ruff format --check --diff --respect-gitignore {posargs:.}
-    ruff: ruff check --respect-gitignore {posargs:.}
+    ruff: ruff format --check --diff {posargs:.}
+    ruff: ruff check {posargs:.}
     shellcheck: xargs -ra {env_tmp_dir}/shellcheck_files shellcheck
     codespell: codespell --toml {tox_root}/pyproject.toml {posargs}
     yaml: yamllint {posargs} .