Merge remote-tracking branch 'upstream/qa/1.x' into ccp

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
         src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json
       )
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.9
+  rev: v0.5.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix]
@@ -45,3 +45,8 @@ repos:
   rev: 054bda51dbe278b3e86f27c890e3f3ac877d616c
   hooks:
     - id: validate-cff
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.10.1
+  hooks:
+  - id: mypy
+    additional_dependencies: [types-all]

hack/Dockerfile

@@ -6,15 +6,16 @@ ARG PYTHON_VERSION=3.9
 ARG NODE_VERSION=20
 ARG PYENV_DIR=/pyenv
 ARG SELENIUM_DIR=/selenium
+ARG MEDIAAREA_VERSION=1.0-24
 
 # -----------------------------------------------------------------------------
 
 FROM ubuntu:${UBUNTU_VERSION} AS base-builder
 
 ARG PYENV_DIR=/pyenv
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
 
 RUN set -ex \
 	&& apt-get update \
@@ -28,14 +29,15 @@ RUN set -ex \
 		libsasl2-dev \
 		libsqlite3-dev \
 		locales \
+		make \
 		pkg-config \
 		tzdata \
 	&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
 RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
 
 ENV PYENV_ROOT=${PYENV_DIR}/data
 ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
@@ -144,17 +146,19 @@ ENTRYPOINT ["npm", "run", "test-single-run"]
 
 # -----------------------------------------------------------------------------
 
-FROM base-builder as base
+FROM base-builder AS base
 
 ARG USER_ID=1000
 ARG GROUP_ID=1000
 ARG PYENV_DIR=/pyenv
+ARG MEDIAAREA_VERSION
 
 RUN set -ex \
 	&& curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \
 	&& echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/archivematica-1.16.x.gpg] http://packages.archivematica.org/1.16.x/ubuntu-externals jammy main" > /etc/apt/sources.list.d/archivematica-external.list \
-	&& curl --retry 3 -so /tmp/repo-mediaarea_1.0-21_all.deb -L https://mediaarea.net/repo/deb/repo-mediaarea_1.0-21_all.deb \
-	&& dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \
+	&& curl --retry 3 -so /tmp/repo-mediaarea.deb -L https://mediaarea.net/repo/deb/repo-mediaarea_${MEDIAAREA_VERSION}_all.deb \
+	&& dpkg -i /tmp/repo-mediaarea.deb \
+	&& rm /tmp/repo-mediaarea.deb \
 	&& apt-get update \
 	&& apt-get install -y --no-install-recommends \
 		atool \
@@ -219,11 +223,11 @@ COPY --chown=${USER_ID}:${GROUP_ID} --link . /src
 
 FROM base AS archivematica-mcp-client
 
-ENV DJANGO_SETTINGS_MODULE settings.common
-ENV PYTHONPATH /src/src/MCPClient/lib/:/src/src/MCPClient/lib/clientScripts:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
-ENV ARCHIVEMATICA_MCPCLIENT_ARCHIVEMATICACLIENTMODULES /src/src/MCPClient/lib/archivematicaClientModules
-ENV ARCHIVEMATICA_MCPCLIENT_CLIENTASSETSDIRECTORY /src/src/MCPClient/lib/assets/
-ENV ARCHIVEMATICA_MCPCLIENT_CLIENTSCRIPTSDIRECTORY /src/src/MCPClient/lib/clientScripts/
+ENV DJANGO_SETTINGS_MODULE=settings.common
+ENV PYTHONPATH=/src/src/MCPClient/lib/:/src/src/MCPClient/lib/clientScripts:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
+ENV ARCHIVEMATICA_MCPCLIENT_ARCHIVEMATICACLIENTMODULES=/src/src/MCPClient/lib/archivematicaClientModules
+ENV ARCHIVEMATICA_MCPCLIENT_CLIENTASSETSDIRECTORY=/src/src/MCPClient/lib/assets/
+ENV ARCHIVEMATICA_MCPCLIENT_CLIENTSCRIPTSDIRECTORY=/src/src/MCPClient/lib/clientScripts/
 
 # Some scripts in archivematica-fpr-admin executed by MCPClient rely on certain
 # files being available in this image (e.g. see https://git.io/vA1wF).
@@ -236,8 +240,8 @@ ENTRYPOINT ["pyenv", "exec", "python3", "/src/src/MCPClient/lib/archivematicaCli
 
 FROM base AS archivematica-mcp-server
 
-ENV DJANGO_SETTINGS_MODULE settings.common
-ENV PYTHONPATH /src/src/MCPServer/lib/:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
+ENV DJANGO_SETTINGS_MODULE=settings.common
+ENV PYTHONPATH=/src/src/MCPServer/lib/:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
 
 ENTRYPOINT ["pyenv", "exec", "python3", "/src/src/MCPServer/lib/archivematicaMCP.py"]
 
@@ -266,17 +270,17 @@ COPY --chown=${USER_ID}:${GROUP_ID} --from=archivematica-dashboard-frontend-buil
 
 WORKDIR /src/src/dashboard/src
 
-ENV DJANGO_SETTINGS_MODULE settings.local
-ENV PYTHONPATH /src/src/dashboard/src/:/src/src/archivematicaCommon/lib/
-ENV AM_GUNICORN_BIND 0.0.0.0:8000
-ENV AM_GUNICORN_CHDIR /src/src/dashboard/src
-ENV FORWARDED_ALLOW_IPS *
+ENV DJANGO_SETTINGS_MODULE=settings.local
+ENV PYTHONPATH=/src/src/dashboard/src/:/src/src/archivematicaCommon/lib/
+ENV AM_GUNICORN_BIND=0.0.0.0:8000
+ENV AM_GUNICORN_CHDIR=/src/src/dashboard/src
+ENV FORWARDED_ALLOW_IPS=*
 
 RUN set -ex \
 	&& ./manage.py collectstatic --noinput --clear \
 	&& ./manage.py compilemessages
 
-ENV DJANGO_SETTINGS_MODULE settings.production
+ENV DJANGO_SETTINGS_MODULE=settings.production
 
 EXPOSE 8000
 

mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+
+explicit_package_bases = True

requirements-dev.txt

@@ -27,7 +27,7 @@ build==1.2.1
     # via pip-tools
 cachetools==5.3.3
     # via tox
-certifi==2024.6.2
+certifi==2024.7.4
     # via
     #   -r requirements.txt
     #   requests
@@ -47,7 +47,7 @@ click==8.1.7
     # via pip-tools
 colorama==0.4.6
     # via tox
-coverage[toml]==7.5.3
+coverage[toml]==7.5.4
     # via
     #   -r requirements-dev.in
     #   pytest-cov
@@ -59,7 +59,7 @@ cryptography==42.0.8
     #   pyopenssl
 distlib==0.3.8
     # via virtualenv
-django==4.2.13
+django==4.2.14
     # via
     #   -r requirements.txt
     #   django-auth-ldap
@@ -86,7 +86,7 @@ elasticsearch==6.8.2
     # via -r requirements.txt
 exceptiongroup==1.2.1
     # via pytest
-filelock==3.15.1
+filelock==3.15.4
     # via
     #   tox
     #   virtualenv
@@ -108,7 +108,7 @@ idna==3.7
     # via
     #   -r requirements.txt
     #   requests
-importlib-metadata==7.1.0
+importlib-metadata==8.0.0
     # via
     #   -r requirements.txt
     #   build
@@ -125,7 +125,7 @@ josepy==1.14.0
     # via
     #   -r requirements.txt
     #   mozilla-django-oidc
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via -r requirements.txt
 jsonschema-specifications==2023.12.1
     # via
@@ -155,7 +155,7 @@ olefile==0.47
     #   opf-fido
 opf-fido @ git+https://github.com/artefactual-labs/fido.git@564ceb8018a8650fe931cf20e6780ee008e60fca
     # via -r requirements.txt
-orjson==3.10.5
+orjson==3.10.6
     # via -r requirements.txt
 packaging==24.1
     # via
@@ -198,7 +198,7 @@ pyopenssl==24.1.0
     # via
     #   -r requirements.txt
     #   josepy
-pyproject-api==1.6.1
+pyproject-api==1.7.1
     # via tox
 pyproject-hooks==1.1.0
     # via
@@ -249,7 +249,7 @@ requests==2.32.3
     #   mozilla-django-oidc
     #   opf-fido
     #   python-cas
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   -r requirements.txt
     #   jsonschema
@@ -272,7 +272,7 @@ tomli==2.0.1
     #   pyproject-api
     #   pytest
     #   tox
-tox==4.15.1
+tox==4.16.0
     # via -r requirements-dev.in
 typing-extensions==4.12.2
     # via
@@ -286,11 +286,11 @@ urllib3==2.2.2
     #   amclient
     #   elasticsearch
     #   requests
-virtualenv==20.26.2
+virtualenv==20.26.3
     # via tox
 wheel==0.43.0
     # via pip-tools
-whitenoise==6.6.0
+whitenoise==6.7.0
     # via -r requirements.txt
 zipp==3.19.2
     # via
@@ -307,9 +307,9 @@ zope-interface==6.4.post2
     #   gevent
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==24.0
+pip==24.1.2
     # via pip-tools
-setuptools==70.0.0
+setuptools==70.3.0
     # via
     #   -r requirements.txt
     #   pip-tools

requirements.txt

@@ -20,7 +20,7 @@ bagit @ git+https://github.com/artefactual-labs/bagit-python.git@4b8fde73b4e6314
     # via -r requirements.in
 brotli==1.1.0
     # via -r requirements.in
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -33,7 +33,7 @@ cryptography==42.0.8
     #   josepy
     #   mozilla-django-oidc
     #   pyopenssl
-django==4.2.13
+django==4.2.14
     # via
     #   -r requirements.in
     #   django-auth-ldap
@@ -70,15 +70,15 @@ gunicorn==22.0.0
     # via -r requirements.in
 idna==3.7
     # via requests
-importlib-metadata==7.1.0
+importlib-metadata==8.0.0
     # via -r requirements.in
 importlib-resources==6.4.0
     # via opf-fido
 inotify-simple==1.3.5
     # via -r requirements.in
 josepy==1.14.0
     # via mozilla-django-oidc
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via -r requirements.in
 jsonschema-specifications==2023.12.1
     # via jsonschema
@@ -100,7 +100,7 @@ olefile==0.47
     # via opf-fido
 opf-fido @ git+https://github.com/artefactual-labs/fido.git@564ceb8018a8650fe931cf20e6780ee008e60fca
     # via -r requirements.in
-orjson==3.10.5
+orjson==3.10.6
     # via -r requirements.in
 packaging==24.1
     # via gunicorn
@@ -144,7 +144,7 @@ requests==2.32.3
     #   mozilla-django-oidc
     #   opf-fido
     #   python-cas
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   jsonschema
     #   referencing
@@ -164,7 +164,7 @@ urllib3==2.2.2
     #   amclient
     #   elasticsearch
     #   requests
-whitenoise==6.6.0
+whitenoise==6.7.0
     # via -r requirements.in
 zipp==3.19.2
     # via
@@ -176,7 +176,7 @@ zope-interface==6.4.post2
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==70.0.0
+setuptools==70.3.0
     # via
     #   zope-event
     #   zope-interface

src/MCPClient/lib/client/job.py

@@ -9,6 +9,7 @@
 import sys
 import traceback
 from contextlib import contextmanager
+from logging.handlers import BufferingHandler
 
 from django.conf import settings
 from django.utils import timezone
@@ -149,7 +150,7 @@ def JobContext(self, logger=None):
                 logger.removeHandler(handler)
 
 
-class JobLogHandler(logging.handlers.BufferingHandler):
+class JobLogHandler(BufferingHandler):
     """
     A handler that buffers log messages, and writes them to Job output
     when the buffer is full.

src/MCPClient/lib/clientScripts/determine_aip_version_key_exit_code.py

@@ -17,11 +17,13 @@
 # along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.
 import os
 import sys
+from typing import Dict
+from typing import Optional
 
 import namespaces as ns
 from lxml import etree
 
-VERSION_MAP = {
+VERSION_MAP: Dict[Optional[str], int] = {
     # Only change exit code if AIP format changes. If unknown, default to latest
     # version. Currently, all AIPs are the same format so no special cases
     # required.

src/MCPClient/lib/clientScripts/fits.py

@@ -34,7 +34,6 @@
 
 logger = get_script_logger("archivematica.mcp.client.FITS")
 
-formats = []
 FITSNS = "{http://hul.harvard.edu/ois/xml/ns/fits/fits_output}"
 
 

src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py

@@ -149,9 +149,9 @@ def main(job):
         # fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail
         # probably already correct, and we only set eventOutcomeDetailNote here
         # Not using .filter().update() because that doesn't generate an exception
-        e = Event.objects.get(event_type="normalization", file_uuid=original_file)
-        e.event_outcome_detail = dstR
-        e.save()
+        event = Event.objects.get(event_type="normalization", file_uuid=original_file)
+        event.event_outcome_detail = dstR
+        event.save()
         job.print_output(
             "Updated the eventOutcomeDetailNote of an existing normalization"
             f" Event for file {fileUUID}. Not creating a Derivation object"