Skip to content

Commit

Permalink
feat: upstream the good parts when rebasing application (#265)
Browse files Browse the repository at this point in the history
* feat: upstream good parts from Makefile/pyproject

* feat(docs): add start of migration instructions

* fix: remove unused variable

* fix: need to set variable to keep uv from complaining

* feat: python-apt launchpad dep in pyproject.toml

* feat: more upstreaming

- Compared .gitignores and added things from my application.
- Removed redundant black things in hook file.
- Ruff format should be after check, as sometimes check's output needs to be
formatted, but format's output shouldn't need to be checked.
- Updated refactor instructions.
- Updated ruff excludes.
- Helped pyright's type checking.

* Apply suggestions from code review

Co-authored-by: Alex Lowe <alex.lowe@canonical.com>

* fix: Alex's PR review

* feat(Makefile): upstream coverage target

* feat(docs): update readme with integration test info

* feat: upstream minor tests workflow things

* fix: uv snap has added aliases and doesn't need beta

* docs: finishing touches on rebase instructions

* fix: the uv.lock file is supposed to be checked in

* feat(docs): make targets called out per @lengau, add update note

* Apply suggestions from code review

* fix(Makefile): upstream npm pyright bugfix

* fix: add in the ruff formatter, per @dariuszd21

* fix: don't shellcheck everything that gets sourced

* Update Makefile

Co-authored-by: Dariusz Duda <darekduda21@gmail.com>

* feat(docs): add note about merging

* feat: allow installing snaps in parallel

Co-authored-by: Alex Lower <alex.lowe@canonical.com>

* Update Makefile

* fix: Darius pointed out that ruff excludes via .gitignore

* fix: `--respect-gitignore` is the default

---------

Co-authored-by: Alex Lowe <alex.lowe@canonical.com>
Co-authored-by: Dariusz Duda <darekduda21@gmail.com>
  • Loading branch information
3 people authored Oct 11, 2024
1 parent 84c2a0e commit 44ce081
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 60 deletions.
13 changes: 10 additions & 3 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,16 @@ on:
- "renovate/*"
pull_request:

# Cancel currently-running builds when pushing new commits to a PR or branch
# https://stackoverflow.com/a/72408109
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
Expand Down Expand Up @@ -42,15 +47,16 @@ jobs:
echo "::group::Wait for snap to complete"
snap watch --last=install
echo "::endgroup::"
- name: Run Linters
- name: Run linters
run: tox run --skip-pkg-install --no-list-dependencies --colored yes -m lint
unit:
strategy:
matrix:
platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python
Expand Down Expand Up @@ -90,6 +96,7 @@ jobs:
platform: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, windows-latest, macos-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Check out code
- uses: actions/checkout@v4
with:
fetch-depth: 0
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
*.snap
MANIFEST

# PyInstaller
Expand Down Expand Up @@ -71,6 +72,7 @@ instance/

# Sphinx documentation
docs/_build/
docs/spelling/

# PyBuilder
target/
Expand Down Expand Up @@ -103,6 +105,7 @@ celerybeat.pid
*.sage.py

# Environments
.direnv
.env
.venv
env/
Expand Down
7 changes: 3 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,11 @@ repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: "v0.6.9"
hooks:
# Run the linter
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- repo: https://github.com/psf/black
rev: "24.8.0"
hooks:
- id: black
# Run the formatter
- id: ruff-format
- repo: https://github.com/adrienverge/yamllint.git
rev: "v1.35.1"
hooks:
Expand Down
Empty file added .shellcheckrc
Empty file.
134 changes: 89 additions & 45 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,44 @@ ifneq ($(OS),Windows_NT)
OS := $(shell uname)
endif

.DEFAULT_GOAL := help

.ONESHELL:

.SHELLFLAGS = -ec

.PHONY: help
help: ## Show this help.
@printf "%-30s %s\n" "Target" "Description"
@printf "%-30s %s\n" "------" "-----------"
@fgrep " ## " $(MAKEFILE_LIST) | fgrep -v grep | awk -F ': .*## ' '{$$1 = sprintf("%-30s", $$1)} 1'
@printf "%-41s %s\n" "Target" "Description"
@printf "%-41s %s\n" "------" "-----------"
@fgrep " ##" $(MAKEFILE_LIST) | fgrep -v grep | sed 's/:[^#]*/ /' | awk -F '[: ]*' \
'{
if ($$2 == "##")
{
$$1=sprintf("%-40s", $$1);
$$2="";
print $$0;
}
else
{
$$1=sprintf(" └%-38s", $$1);
$$2="";
print $$0;
}
}'

---------------- : ## ----------------

.PHONY: setup
setup: ## Set up a development environment
ifeq ($(OS),Linux)
sudo snap install codespell ruff shellcheck
sudo snap install --classic --beta astral-uv
sudo snap alias astral-uv.uv uv
sudo snap alias astral-uv.uvx uvx
changes="`sudo snap install --no-wait codespell`"
changes="${changes} `sudo snap install --no-wait ruff`"
changes="${changes} `sudo snap install --no-wait shellcheck`"
changes="${changes} `sudo snap install --classic --no-wait astral-uv`"
for change in ${changes}; do
snap watch ${change}
done
else ifeq ($(OS),Windows_NT)
pipx install uv
choco install shellcheck
Expand All @@ -39,68 +64,87 @@ ifeq (, $(shell which pre-commit))
endif
pre-commit install

---------------- : ## ----------------

.PHONY: autoformat
autoformat: format-ruff format-codespell ## Run all automatic formatters

.PHONY: lint
lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs ## Run all linters

.PHONY: test
test: test-unit test-integration ## Run all tests

.PHONY: docs
docs: ## Build documentation
uv run --extra docs sphinx-build -b html -W docs docs/_build
.PHONY: format-ruff
format-ruff: ##- Automatically format with ruff
ruff check --fix $(SOURCES)
success=true
ruff check --fix $(SOURCES) || success=false
ruff format $(SOURCES)
$success || exit 1

.PHONY: docs-auto
docs-auto: ## Build and host docs with sphinx-autobuild
uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build
.PHONY: format-codespell
format-codespell: ##- Fix spelling issues with codespell
uv run codespell --toml pyproject.toml --write-changes $(SOURCES)

# Helpful in `help` to split the main targets from things that build
---------------- : ## ----------------

.PHONY: format-codespell
format-codespell: ## Fix spelling issues with codespell
uv run codespell --toml pyproject.toml --write-changes $(SOURCES)
.PHONY: lint
lint: lint-ruff lint-codespell lint-mypy lint-pyright lint-shellcheck lint-yaml lint-docs ## Run all linters

.PHONY: format-ruff
format-ruff: ## Automatically format with ruff
ruff format $(SOURCES)
ruff check --fix $(SOURCES)
.PHONY: lint-ruff
lint-ruff: ##- Lint with ruff
ruff check $(SOURCES)
ruff format --diff $(SOURCES)

.PHONY: lint-codespell
lint-codespell: ## Check spelling with codespell
lint-codespell: ##- Check spelling with codespell
uv run codespell --toml pyproject.toml $(SOURCES)

.PHONY: lint-docs
lint-docs: ## Lint the documentation
uv run --extra docs sphinx-lint --max-line-length 80 --enable all $(DOCS)

.PHONY: lint-mypy
lint-mypy: ## Check types with mypy
uv run mypy $(SOURCES)
lint-mypy: ##- Check types with mypy
uv run mypy --show-traceback --show-error-codes $(SOURCES)

.PHONY: lint-pyright
lint-pyright: ## Check types with pyright
lint-pyright: ##- Check types with pyright
# Fix for a bug in npm
[ -d "/home/ubuntu/.npm/_cacache" ] && chown -R 1000:1000 "/home/ubuntu/.npm" || true
uv run pyright

.PHONY: lint-ruff
lint-ruff: ## Lint with ruff
ruff format --diff $(SOURCES)
ruff check $(SOURCES)

.PHONY: lint-shellcheck
lint-shellcheck:
sh -c 'git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck'
lint-shellcheck: ##- Lint shell scripts
git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs -r shellcheck

.PHONY: lint-yaml
lint-yaml: ## Lint YAML files with yamllint
lint-yaml: ##- Lint YAML files with yamllint
uv run yamllint .

.PHONY: lint-docs
lint-docs: ##- Lint the documentation
uv run --extra docs sphinx-lint --max-line-length 88 --enable all $(DOCS)

---------------- : ## ----------------

.PHONY: test
test: test-unit test-integration ## Run all tests

.PHONY: test-unit
test-unit: ## Run unit tests
test-unit: ##- Run unit tests
uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.unit.xml --junit-xml=.results.unit.xml tests/unit

.PHONY: test-integration
test-integration: ## Run integration tests
test-integration: ##- Run integration tests
uv run pytest --cov=$(PROJECT) --cov-config=pyproject.toml --cov-report=xml:.coverage.integration.xml --junit-xml=.results.integration.xml tests/integration

---------------- : ## ----------------

.PHONY: coverage
coverage: ## Generate coverage report
coverage run --source starcraft -m pytest
coverage xml -o coverage.xml
coverage report -m
coverage html

---------------- : ## ----------------

.PHONY: docs
docs: ## Build documentation
uv run --extra docs sphinx-build -b html -W docs docs/_build

.PHONY: docs-auto
docs-auto: ## Build and host docs with sphinx-autobuild
uv run --extra docs sphinx-autobuild -b html --open-browser --port=8080 --watch $(PROJECT) -W docs docs/_build
59 changes: 58 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,64 @@ TODO

Migrate existing projects
--------------------------------
TODO
#. Update this guide as you go along, if something is unclear or missing.
#. Use ruff.
#. Pull in the bare minimum ``pyproject.toml`` needed to use ruff.
#. Make your codebase pass with ruff. Commit after each step:
#. ``ruff check --fix``
#. ``ruff check --fix --unsafe-fixes``
#. ``ruff check --add-noqa``
#. ``ruff format``
#. Replace use of black, flake8, pydocstyle, isort, and pylint in Makefile/CI
with:
- ``ruff check --fix``
- ``ruff format``
#. Modify top-level files in your project to match what's in Starbase as closely
as possible.
#. ``Makefile`` - Ensure you use ``uv`` and at least have the same targets:
- ``setup``
- ``lint``
- ``test-unit``
- ``test-integration`` (If this applies to your repo, i.e. the repo is a library
rather than an application)
- ``coverage``
#. ``pyproject.toml`` - Expand from just the ruff things: move things into
here from your ``setup.py``, ``setup.cfg``, and ``requirements.*.txt``.
#. ``README`` - If your readme is .md, convert to .rst with pandoc:
``pandoc -o README.rst README.md``
Don't worry about making the contents match, Starbase's is very specific.
#. Run all the linters: ``make lint``
#. ``mypy``:
- Mypy checks the same things as ``ruff``'s ``ANNXXX`` checks, but
``ruff``'s ``noqa`` directives mean nothing to mypy. You'll need to fix
these by hand, mostly by adding type annotations to function definitions.
#. ``pyright``:
- For errors along the lines of "Stub file not found for $library", check
for the existence of pip package ``typing-$library`` and add it as a
dependency.
- If you have lots of errors you may need to remove the ``strict``
directive from ``pyproject.toml``.
#. Do a side-by-side diff of the ``.gitignore`` files in your project and
Starbase, making them as close as possible and adding anything that makes
sense upstream.
#. Bring in remaining top-level files:
- .editorconfig
- .pre-commit-config.yaml
- .shellcheckrc
- tox.ini
- .yamllint.yaml
#. If you're rebasing a library, add the integrations tests structure.
Applications should use spread for integration tests.
# Finally, once all files are manually synced, actually sync the git history:
- ``git remote add starbase git@github.com:canonical/starbase.git``
- ``git merge --allow-unrelated-histories starbase/main``
- ``git remote remove starbase``
- Don't forget to review all the new files and dirs that this merge adds -
you'll want to delete a lot of them.
- When you merge, DO NOT squash, otherwise the starbase history will not be
preserved.



Create a new project
---------------------------
Expand Down
9 changes: 4 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[project]
name = "starcraft"
description = "The basis for Starcraft team packages"
dynamic = ["version", "readme"]
dependencies = [

Expand Down Expand Up @@ -157,10 +158,7 @@ strict = false
line-length = 88
target-version = "py310"
src = ["starcraft", "tests"]
extend-exclude = [
"docs",
"__pycache__",
]
extend-exclude = ["docs"]

[tool.ruff.format]
docstring-code-format = true
Expand All @@ -169,6 +167,7 @@ quote-style = "double"

[tool.ruff.lint]
# Follow ST063 - Maintaining and updating linting specifications for updating these.
# Handy link: https://docs.astral.sh/ruff/rules/
select = [ # Base linting rule selections.
# See the internal document for discussion:
# https://docs.google.com/document/d/1i1n8pDmFmWi4wTDpk-JfnWCVUThPJiggyPi2DYwBBu4/edit
Expand Down Expand Up @@ -285,7 +284,7 @@ max-args = 8
classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"]

[tool.ruff.lint.per-file-ignores]
"tests/**.py" = [ # Some things we want for the moin project are unnecessary in tests.
"tests/**.py" = [ # Some things we want for the main project are unnecessary in tests.
"D", # Ignore docstring rules in tests
"ANN", # Ignore type annotations in tests
"ARG", # Allow unused arguments in tests (e.g. for fake functions/methods/classes)
Expand Down
4 changes: 2 additions & 2 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ labels = lint
commands_pre =
shellcheck: bash -c '{[shellcheck]find} | {[shellcheck]filter} > {env_tmp_dir}/shellcheck_files'
commands =
ruff: ruff format --check --diff --respect-gitignore {posargs:.}
ruff: ruff check --respect-gitignore {posargs:.}
ruff: ruff format --check --diff {posargs:.}
ruff: ruff check {posargs:.}
shellcheck: xargs -ra {env_tmp_dir}/shellcheck_files shellcheck
codespell: codespell --toml {tox_root}/pyproject.toml {posargs}
yaml: yamllint {posargs} .
Expand Down

0 comments on commit 44ce081

Please sign in to comment.