IN 1059 - Maintenance 2024-08

.gitignore

@@ -131,4 +131,6 @@ dmypy.json
 .DS_Store
 cov_html
 .vscode/
-.idea/
+.idea/
+
+output/

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+default_language_version:
+  python: python3.11 # set for project python version
+repos:
+  - repo: local
+    hooks:
+      - id: black-apply
+        name: black-apply
+        entry: pipenv run black
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: mypy
+        name: mypy
+        entry: pipenv run mypy
+        language: system
+        pass_filenames: true
+        types: ["python"]
+        exclude: "tests/"
+      - id: ruff-apply
+        name: ruff-apply
+        entry: pipenv run ruff check --fix
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: safety
+        name: safety
+        entry: pipenv check
+        language: system
+        pass_filenames: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim as build
+FROM python:3.12-slim as build
 WORKDIR /app
 COPY . .
 RUN cd /app && python setup.py bdist_wheel

Makefile

@@ -1,3 +1,62 @@
+SHELL=/bin/bash
+DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
+
+help: # Preview Makefile commands
+	@awk 'BEGIN { FS = ":.*#"; print "Usage:  make <target>\n\nTargets:" } \
+/^[-_[:alpha:]]+:.?*#/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
+#######################
+# Dependency commands
+#######################
+
+install: # Install Python dependencies
+	pipenv install --dev
+	pipenv run pre-commit install
+
+update: install # Update Python dependencies
+	pipenv clean
+	pipenv update --dev
+
+
+######################
+# Unit test commands
+######################
+
+test: # Run tests and print a coverage report
+	pipenv run coverage run --source=submitter -m pytest -vv
+	pipenv run coverage report -m
+
+coveralls: test # Write coverage data to an LCOV report
+	pipenv run coverage lcov -o ./coverage/lcov.info
+
+
+####################################
+# Code quality and safety commands
+####################################
+
+lint: black mypy ruff safety # Run linters
+
+black: # Run 'black' linter and print a preview of suggested changes
+	pipenv run black --check --diff .
+
+mypy: # Run 'mypy' linter
+	pipenv run mypy .
+
+ruff: # Run 'ruff' linter and print a preview of errors
+	pipenv run ruff check .
+
+safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-date
+	pipenv check
+	pipenv verify
+
+lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
+
+black-apply: # Apply changes with 'black'
+	pipenv run black .
+
+ruff-apply: # Resolve 'fixable errors' with 'ruff'
+	pipenv run ruff check --fix .
+
 ### This is the Terraform-generated header for dspace-submission-service-dev. If  ###
 ###   this is a Lambda repo, uncomment the FUNCTION line below  ###
 ###   and review the other commented lines in the document.     ###
@@ -65,37 +124,4 @@ verify-dspace-connection-prod: # Verify prod app can connect to DSpace
 #   Not yet deployed in production
 
 # verify-dspace-connection-prod: # Verify prod app can connect to DSpace
-# 	Not yet deployed in production
-
-### Dependency commands ###
-install: ## Install script and dependencies
-	pipenv install --dev
-
-update: install ## Update all Python dependencies
-	pipenv clean
-	pipenv update --dev
-
-
-### Testing commands ###
-test: ## Run tests and print a coverage report
-	pipenv run coverage run --source=submitter -m pytest -vv
-	pipenv run coverage report -m
-
-coveralls: test
-	pipenv run coverage lcov -o ./coverage/lcov.info
-
-
-### Linting commands ###
-lint: bandit black flake8 isort ## Lint the repo
-
-bandit:
-	pipenv run bandit -r submitter
-
-black:
-	pipenv run black --check --diff .
-
-flake8:
-	pipenv run flake8 .
-
-isort:
-	pipenv run isort . --diff
+# 	Not yet deployed in production

Pipfile

@@ -11,10 +11,7 @@ sentry-sdk = "*"
 smart-open = "*"
 
 [dev-packages]
-flake8 = "*"
 black = "*"
-isort = "*"
-bandit = "*"
 moto = {extras = ["s3", "server", "sqs"], version = "*"}
 pytest = "*"
 coveralls = "*"
@@ -23,9 +20,12 @@ pytest-cov = "*"
 pytest-env = "*"
 freezegun = "*"
 coverage = "*"
+ruff = "*"
+safety = "*"
+pre-commit = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"
 
 [scripts]
 submitter = "python -c \"from submitter.cli import main; main()\""

pyproject.toml

@@ -1,8 +1,73 @@
-[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta:__legacy__"
+[tool.black]
+line-length = 90
+
+[tool.mypy]
+disallow_untyped_calls = true
+disallow_untyped_defs = true
+exclude = ["tests/"]
 
 [tool.pytest.ini_options]
+log_level = "INFO"
 env = [
   "WORKSPACE=test"
 ]
+
+[tool.ruff]
+target-version = "py312"
+
+# set max line length
+line-length = 90
+
+# enumerate all fixed violations
+show-fixes = true
+
+[tool.ruff.lint]
+select = ["ALL", "PT"]
+ignore = [
+    # default
+    "ANN101",
+    "ANN102",
+    "COM812",
+    "D107",
+    "N812",
+    "PTH",
+
+    # project-specific
+    "C90",
+    "D100",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
+    "PLR0912",
+    "PLR0913",
+    "PLR0915",
+    "S320",
+    "S321",
+]
+
+# allow autofix behavior for specified rules
+fixable = ["E", "F", "I", "Q"]
+
+[tool.ruff.lint.flake8-annotations]
+mypy-init-return = true
+
+[tool.ruff.lint.flake8-pytest-style]
+fixture-parentheses = false
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*" = [
+    "ANN",
+    "ARG001",
+    "S101",
+]
+
+[tool.ruff.lint.pycodestyle]
+max-doc-length = 90
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta:__legacy__"

tests/conftest.py

@@ -5,7 +5,7 @@
 import pytest
 import requests_mock
 from dspace import DSpaceClient
-from moto import mock_iam, mock_s3, mock_sqs
+from moto import mock_aws
 from requests import exceptions
 
 
@@ -19,7 +19,7 @@ def aws_credentials():
 
 @pytest.fixture()
 def test_aws_user(aws_credentials):
-    with mock_iam():
+    with mock_aws():
         user_name = "test-user"
         policy_document = {
             "Version": "2012-10-17",
@@ -119,7 +119,7 @@ def mocked_dspace_auth_failure():
 
 @pytest.fixture(scope="function")
 def mocked_sqs(aws_credentials):
-    with mock_sqs():
+    with mock_aws():
         sqs = boto3.resource("sqs")
         sqs.create_queue(QueueName="empty_input_queue")
         sqs.create_queue(QueueName="empty_result_queue")
@@ -152,7 +152,7 @@ def mocked_sqs(aws_credentials):
 
 @pytest.fixture()
 def mocked_s3(aws_credentials):
-    with mock_s3():
+    with mock_aws():
         s3 = boto3.client("s3")
         s3.create_bucket(
             Bucket="test-bucket",