From 30c9805718f93d3cd2dde64977a84f5a6203b157 Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Wed, 8 Nov 2023 14:58:49 -0600 Subject: [PATCH] Drop unsupported Python versions * Upgrade pre-commit dependencies * Rename codecov config file * Update manifest file * Update tests configuration * Update contributor's agreement link * Do not create universal wheels * Migrate to pyproject.toml * Update requirements files * Run docs CI job separately * Pin sphinx version to support Python 3.8 * Add make rules for pip-compile commands * Add GitHub workflow for uploading to PyPI * Bump version --- codecov.yml => .codecov.yml | 0 .coveragerc | 6 -- .flake8 | 16 ++++ .github/workflows/release.yml | 38 ++++++++++ .github/workflows/test.yml | 61 +++++++-------- .pre-commit-config.yaml | 19 ++--- CONTRIBUTING.md | 4 +- MANIFEST.in | 3 +- Makefile | 34 ++++++--- metsrw/__init__.py | 2 +- metsrw/fsentry.py | 6 +- metsrw/plugins/premisrw/premis.py | 4 +- pyproject.toml | 118 ++++++++++++++++++++++++++++++ requirements-dev.txt | 108 +++++++++++++++++++++++++++ requirements.txt | 8 ++ requirements/base.txt | 1 - requirements/dev.txt | 5 -- setup.py | 60 --------------- tox.ini | 25 ------- 19 files changed, 358 insertions(+), 160 deletions(-) rename codecov.yml => .codecov.yml (100%) delete mode 100644 .coveragerc create mode 100644 .flake8 create mode 100644 .github/workflows/release.yml create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt delete mode 100644 requirements/base.txt delete mode 100644 requirements/dev.txt delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/codecov.yml b/.codecov.yml similarity index 100% rename from codecov.yml rename to .codecov.yml diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 3fc3736..0000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[run] -branch = True -source = metsrw - -[report] -omit = diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..28052a5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,16 @@ +[flake8] +exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs +# Error codes: +# - https://flake8.pycqa.org/en/latest/user/error-codes.html +# - https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +# - https://github.com/PyCQA/flake8-bugbear#list-of-warnings +# +# E203: whitespace before `,`, `;` or `:` +# E402: module level import not at top of file +# E501: line too long +# W503: line break before binary operator +ignore = + E203, + E402, + E501, + W503 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d2c8ed7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +--- +name: "Release" +on: "workflow_dispatch" +jobs: + build: + name: "Build" + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository" + uses: "actions/checkout@v4" + - name: "Set up Python" + uses: "actions/setup-python@v4" + with: + python-version: "3.9" + - name: "Build distribution packages" + run: make package-check + - name: "Save distribution directory" + uses: "actions/upload-artifact@v3" + with: + name: "distribution" + path: | + dist + upload: + name: "Upload" + needs: "build" + runs-on: "ubuntu-22.04" + environment: "release" + permissions: + id-token: "write" + steps: + - name: "Restore distribution directory" + uses: "actions/download-artifact@v3" + with: + name: "distribution" + path: | + dist + - name: "Upload distribution packages to PyPI" + uses: "pypa/gh-action-pypi-publish@release/v1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3097733..2df6422 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,65 +7,57 @@ on: - "master" jobs: test: - name: "Test ${{ matrix.toxenv }}" - runs-on: "ubuntu-20.04" + name: "Test Python ${{ matrix.python-version }}" + runs-on: "ubuntu-22.04" strategy: + fail-fast: false matrix: - include: - - python-version: "3.6" - toxenv: "py36" - - python-version: "3.7" - toxenv: "py37" - - python-version: "3.8" - toxenv: "py38" - - python-version: "3.9" - toxenv: "py39" + python-version: [ + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + ] steps: - name: "Check out repository" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python ${{ matrix.python-version }}" uses: "actions/setup-python@v4" with: python-version: "${{ matrix.python-version }}" - - name: "Get pip cache dir" - id: "pip-cache" - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - name: "Cache pip packages" - uses: "actions/cache@v3" - with: - path: "${{ steps.pip-cache.outputs.dir }}" - key: "${{ runner.os }}-pip-${{ hashFiles('**/base.txt', '**/dev.txt') }}" - restore-keys: | - ${{ runner.os }}-pip- + cache: "pip" + cache-dependency-path: | + requirements.txt + requirements-dev.txt - name: "Install tox" run: | python -m pip install --upgrade pip - pip install tox + pip install tox tox-gh-actions - name: "Run tox" - env: - TOXENV: ${{ matrix.toxenv }} run: | - tox -- --cov-report xml:coverage.xml + tox -- --cov metsrw --cov-report xml:coverage.xml - name: "Upload coverage report" if: github.repository == 'artefactual-labs/mets-reader-writer' uses: "codecov/codecov-action@v3" with: files: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false verbose: true - name: ${{ matrix.toxenv }} - flags: ${{ matrix.toxenv }} lint: name: "Lint" runs-on: "ubuntu-22.04" steps: - name: "Check out repository" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.12" + cache: "pip" + cache-dependency-path: | + requirements.txt + requirements-dev.txt - name: "Install tox" run: | python -m pip install --upgrade pip @@ -82,7 +74,10 @@ jobs: - name: "Set up Python" uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.12" + cache-dependency-path: | + requirements.txt + requirements-dev.txt - name: "Install tox" run: | python -m pip install --upgrade pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db6b571..07bab0c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,23 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + rev: v3.15.0 hooks: - id: pyupgrade - args: [--py3-plus, --py36-plus] + args: [--py38-plus] - repo: https://github.com/asottile/reorder_python_imports - rev: v2.6.0 + rev: v3.12.0 hooks: - id: reorder-python-imports - args: [--py3-plus, --py36-plus] -- repo: https://github.com/ambv/black - rev: 22.8.0 + args: [--py38-plus] +- repo: https://github.com/psf/black + rev: "23.10.1" hooks: - id: black args: [--safe, --quiet] - language_version: python3 - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + rev: "6.1.0" hooks: - id: flake8 - language_version: python3 + additional_dependencies: + - flake8-bugbear==23.9.16 + - flake8-comprehensions==3.14.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b83c71..3d4c5f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ For more information on contribution guidelines and standards, see the CONTRIBUT ## Contributor's Agreement -In order for the Archivematica development team to accept any patches or code commits, contributors must first sign this [Contributor's Agreement](https://wiki.archivematica.org/images/2/25/Contributor_agreement.txt). +In order for the Archivematica development team to accept any patches or code commits, contributors must first sign this [Contributor's Agreement](https://wiki.archivematica.org/images/e/e6/Archivematica-CLA-firstname-lastname-YYYY.pdf). The Archivematica contributor's agreement is based almost verbatim on the [Apache Foundation](http://apache.org )'s individual [contributor license](http://www.apache.org/licenses/icla.txt). If you have any questions or concerns about the Contributor's Agreement, please email us at agreement@artefactual.com to discuss them. @@ -61,7 +61,7 @@ This ensures our resources are devoted to making our project the best they can b ### How do I send in an agreement? -Please print out, read, sign, and scan the [contributor agreement](https://wiki.archivematica.org/images/2/25/Contributor_agreement.txt) and email it to agreement@artefactual.com +Please print out, read, sign, and scan the [contributor agreement](https://wiki.archivematica.org/images/e/e6/Archivematica-CLA-firstname-lastname-YYYY.pdf) and email it to agreement@artefactual.com Alternatively, you may send an original signed agreement to: Artefactual Systems Inc. diff --git a/MANIFEST.in b/MANIFEST.in index f55fc27..0cd4735 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include metsrw/resources/* -include LICENSE \ No newline at end of file +include LICENSE +include README.md diff --git a/Makefile b/Makefile index a86ec4c..4c0a129 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,33 @@ -.PHONY: clean package package-deps package-source package-upload package-wheel +.DEFAULT_GOAL := help -package-deps: - pip install --upgrade twine wheel +.PHONY: clean package package-deps package-distribution package-upload pip-compile pip-upgrade -package-source: - python setup.py sdist +package-deps: ## Upgrade dependencies for packaging + python3 -m pip install --upgrade build twine -package-wheel: package-deps - python setup.py bdist_wheel --universal +package-distribution: package-deps ## Create distribution packages + python3 -m build -package-check: package-source package-wheel ## Check the distribution is valid - twine check dist/* +package-check: package-distribution ## Check the distribution is valid + python3 -m twine check --strict dist/* -package-upload: package-deps package-check - twine upload dist/* --repository-url https://upload.pypi.org/legacy/ +package-upload: package-deps package-check ## Upload distribution packages + python3 -m twine upload dist/* --repository-url https://upload.pypi.org/legacy/ package: package-upload -clean: +clean: ## Clean the package directory rm -rf metsrw.egg-info/ rm -rf build/ rm -rf dist/ + +pip-compile: ## Compile pip requirements + pip-compile --allow-unsafe --output-file=requirements.txt pyproject.toml + pip-compile --allow-unsafe --extra=dev --output-file=requirements-dev.txt pyproject.toml + +pip-upgrade: ## Upgrade pip requirements + pip-compile --allow-unsafe --upgrade --output-file=requirements.txt pyproject.toml + pip-compile --allow-unsafe --upgrade --extra=dev --output-file=requirements-dev.txt pyproject.toml + +help: ## Print this help message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/metsrw/__init__.py b/metsrw/__init__.py index 09a3e5c..b6f119c 100644 --- a/metsrw/__init__.py +++ b/metsrw/__init__.py @@ -41,7 +41,7 @@ LOGGER = logging.getLogger(__name__) LOGGER.addHandler(logging.NullHandler()) -__version__ = "0.4.0" +__version__ = "0.5.0" __all__ = [ "Agent", diff --git a/metsrw/fsentry.py b/metsrw/fsentry.py index b5a1729..b7f04bf 100644 --- a/metsrw/fsentry.py +++ b/metsrw/fsentry.py @@ -298,9 +298,9 @@ def add_dmdsec(self, md, mdtype, mode="mdwrap", **kwargs): dmdsec.status = kwargs.get("status") or "original" mdtype_key = utils.generate_mdtype_key(mdtype, kwargs.get("othermdtype", "")) if mdtype_key in self.dmdsecs_by_mdtype: - group_id = getattr(self.dmdsecs_by_mdtype[mdtype_key][0], "group_id") - if not group_id: - group_id = str(uuid4()) + group_id = getattr( + self.dmdsecs_by_mdtype[mdtype_key][0], "group_id", str(uuid4()) + ) dmdsec.group_id = group_id for previous_dmdsec in self.dmdsecs_by_mdtype[mdtype_key]: previous_dmdsec.group_id = group_id diff --git a/metsrw/plugins/premisrw/premis.py b/metsrw/plugins/premisrw/premis.py index 976ec81..bd1ea5d 100644 --- a/metsrw/plugins/premisrw/premis.py +++ b/metsrw/plugins/premisrw/premis.py @@ -178,7 +178,7 @@ def __getattr__(self, attr_name): set( list(self.attrs_to_paths.keys()) + [x.replace("/", "__") for x in self.attrs_to_paths.values()] - + list(x.replace(":", "_") for x in self.attributes.keys()) + + [x.replace(":", "_") for x in self.attributes.keys()] ) ) ) @@ -897,7 +897,7 @@ def _generate_data(schema, elements, attributes=None, path=None): subel = _generate_data(subschema, elements, path=new_path) if (not subel) or (subel == subschema): continue - if all(map(lambda x: isinstance(x, tuple), subel)): + if all(isinstance(x, tuple) for x in subel): for subsubel in subel: data.append(subsubel) elif not el_is_empty(subel): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ebbd2ee --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,118 @@ +[build-system] +requires = [ + "setuptools>=68", + "wheel>=0.41", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = [""] +include = ["metsrw*"] +namespaces = false + +[project] +name = "metsrw" +dynamic = [ + "version", + "readme", +] +description = "Library for dealing with METS files." +requires-python = ">=3.8" +license = {file = "LICENSE"} +dependencies = [ + "lxml", +] +keywords = [ + "archivematica", + "preservation", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +authors = [ + {name = "Artefactual Systems Inc.", email = "info@artefactual.com"} +] +maintainers = [ + {name = "Artefactual Systems Inc.", email = "info@artefactual.com"} +] + +[project.urls] +homepage = "https://github.com/artefactual-labs/mets-reader-writer/" +documentation = "https://mets-reader-writer.readthedocs.io/" +repository = "https://github.com/artefactual-labs/mets-reader-writer/" +issues = "https://github.com/archivematica/Issues/issues" + +[project.optional-dependencies] +dev = [ + "coverage", + "pip-tools", + "pytest-cov", + "pytest", + "sphinx==7.1.2", + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", +] + +[tool.setuptools.dynamic] +version = {attr = "metsrw.__version__"} +readme = {file = ["README.md"], content-type = "text/markdown"} + +[tool.pytest.ini_options] +python_files = [ + "test_*.py", +] +testpaths = [ + "tests", +] + +[tool.coverage.run] +source = [ + "metsrw", +] +branch = true +omit = [ + "tests/*", +] + +[tool.tox] +legacy_tox_ini = """ + [tox] + envlist = py{38,39,310,311,312}, linting, docs + + [gh-actions] + python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 + + [testenv] + skip_install = true + deps = -r {toxinidir}/requirements-dev.txt + commands = pytest {posargs} + + [testenv:linting] + basepython = python3 + deps = pre-commit + commands = pre-commit run --all-files --show-diff-on-failure + + [testenv:docs] + changedir = docs + commands = + sphinx-build -WT -b doctest -d {envtmpdir}/doctrees . {envtmpdir}/html + sphinx-build -WT -b dummy -d {envtmpdir}/doctrees . {envtmpdir}/html +""" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..f75810a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,108 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --extra=dev --output-file=requirements-dev.txt pyproject.toml +# +alabaster==0.7.13 + # via sphinx +babel==2.13.1 + # via sphinx +build==1.0.3 + # via pip-tools +certifi==2023.7.22 + # via requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via pip-tools +coverage[toml]==7.3.2 + # via + # metsrw (pyproject.toml) + # pytest-cov +docutils==0.20.1 + # via sphinx +exceptiongroup==1.1.3 + # via pytest +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==6.8.0 + # via + # build + # sphinx +iniconfig==2.0.0 + # via pytest +jinja2==3.1.2 + # via sphinx +lxml==4.9.3 + # via metsrw (pyproject.toml) +markupsafe==2.1.3 + # via jinja2 +packaging==23.2 + # via + # build + # pytest + # sphinx +pip-tools==7.3.0 + # via metsrw (pyproject.toml) +pluggy==1.3.0 + # via pytest +pygments==2.16.1 + # via sphinx +pyproject-hooks==1.0.0 + # via build +pytest==7.4.3 + # via + # metsrw (pyproject.toml) + # pytest-cov +pytest-cov==4.1.0 + # via metsrw (pyproject.toml) +requests==2.31.0 + # via sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.1.2 + # via metsrw (pyproject.toml) +sphinxcontrib-applehelp==1.0.4 + # via + # metsrw (pyproject.toml) + # sphinx +sphinxcontrib-devhelp==1.0.2 + # via + # metsrw (pyproject.toml) + # sphinx +sphinxcontrib-htmlhelp==2.0.1 + # via + # metsrw (pyproject.toml) + # sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via + # metsrw (pyproject.toml) + # sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via + # metsrw (pyproject.toml) + # sphinx +tomli==2.0.1 + # via + # build + # coverage + # pip-tools + # pyproject-hooks + # pytest +urllib3==2.0.7 + # via requests +wheel==0.41.3 + # via pip-tools +zipp==3.17.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +pip==23.3.1 + # via pip-tools +setuptools==68.2.2 + # via pip-tools diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed84951 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=requirements.txt pyproject.toml +# +lxml==4.9.3 + # via metsrw (pyproject.toml) diff --git a/requirements/base.txt b/requirements/base.txt deleted file mode 100644 index ab90481..0000000 --- a/requirements/base.txt +++ /dev/null @@ -1 +0,0 @@ -lxml diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index d3a09bb..0000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,5 +0,0 @@ --r base.txt -pytest -pytest-cov -sphinx>=1.3 -tox diff --git a/setup.py b/setup.py deleted file mode 100644 index 5939a13..0000000 --- a/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -"""A setuptools based setup module. - -See: -https://packaging.python.org/en/latest/distributing.html -https://github.com/pypa/sampleproject -""" -import codecs -import re -from os import path - -from setuptools import find_packages -from setuptools import setup - -here = path.abspath(path.dirname(__file__)) - - -def read(*parts): - with codecs.open(path.join(here, *parts), "r") as fp: - return fp.read() - - -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) - if version_match: - return version_match.group(1) - raise RuntimeError("Unable to find version string.") - - -# Get the long description from the relevant file -with codecs.open(path.join(here, "README.md"), encoding="utf-8") as f: - long_description = f.read() - - -setup( - name="metsrw", - version=find_version("metsrw", "__init__.py"), - description="Library for dealing with METS files.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/artefactual-labs/mets-reader-writer/", - author="Artefactual", - author_email="info@artefactual.com", - license="AGPL", - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - ], - keywords="mets", - packages=find_packages(exclude=["docs", "fixtures", "requirements", "tests*"]), - install_requires=["lxml"], - include_package_data=True, - python_requires=">=3.6", -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b180e35..0000000 --- a/tox.ini +++ /dev/null @@ -1,25 +0,0 @@ -[tox] -envlist = py{36,37,38,39}, linting, docs -skip_missing_interpreters = True -skipsdist = True - -[testenv] -deps = -rrequirements/dev.txt -commands = pytest --cov-config .coveragerc --cov metsrw {posargs} - -[testenv:linting] -basepython = python3 -deps = pre-commit -commands = pre-commit run --all-files --show-diff-on-failure - -[testenv:docs] -changedir = docs -commands = - sphinx-build -WT -b doctest -d {envtmpdir}/doctrees . {envtmpdir}/html - sphinx-build -WT -b dummy -d {envtmpdir}/doctrees . {envtmpdir}/html - -[flake8] -exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs -application-import-names = flake8 -select = C, E, F, W, B, B950 -ignore = E501, W503