Merge pull request #42 from MITLibraries/IN-1059-maintenance-08-2024

IN 1059 - Maintenance 2024-08

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,28 +1,29 @@
-#### Includes new or updated dependencies?
+### Purpose and background context
+Describe the overall purpose of the PR changes and any useful background context.
 
-YES | NO
+### How can a reviewer manually see the effects of these changes?
+Explain how to see the proposed changes in the application if possible.
 
-#### Changes expectations for external applications?
+Delete this section if it isn't applicable to the PR.
 
+### Includes new or updated dependencies?
 YES | NO
 
-#### Developer
+### Changes expectations for external applications?
+YES | NO
 
+### What are the relevant tickets?
+- Include links to Jira Software and/or Jira Service Management tickets here.
+
+### Developer
 - [ ] All new ENV is documented in README
 - [ ] All new ENV has been added to staging and production environments
 - [ ] All related Jira tickets are linked in commit message(s)
+- [ ] Stakeholder approval has been confirmed (or is not needed)
 
-#### How can a reviewer manually see the effects of these changes?
-
-Explain how to see the proposed changes in the application.
-
-Delete this section if it isn't applicable to the PR.
-
-#### Code Reviewer
-
-- [ ] The commit message is clear and follows our guidelines
-      (not just this pull request message)
+### Code Reviewer(s)
+- [ ] The commit message is clear and follows our guidelines (not just this PR message)
 - [ ] There are appropriate tests covering any new functionality
-- [ ] The documentation has been updated or is unnecessary
-- [ ] The changes have been verified
+- [ ] The provided documentation is sufficient for understanding any new functionality introduced
+- [ ] Any manual tests have been performed **or** provided examples verified
 - [ ] New dependencies are appropriate or there were no changes

.github/workflows/ci.yml

@@ -1,41 +1,7 @@
-name: Tests
+name: CI
 on: push
 jobs:
   test:
-    name: Run tests
-    env:
-      COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-      - name: Install
-        run: |
-          python -m pip install --upgrade pip pipenv
-          pipenv install --dev
-      - name: Tests
-        run: make coveralls
-  linting:
-    name: Run linting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-      - name: Install
-        run: |
-          python -m pip install --upgrade pip pipenv
-          pipenv install --dev
-      - name: bandit
-        run: make bandit
-      - name: black
-        run: make black
-      - name: flake8
-        run: make flake8
-      - name: isort
-        run: make isort
+    uses: mitlibraries/.github/.github/workflows/python-shared-test.yml@main
+  lint:
+    uses: mitlibraries/.github/.github/workflows/python-shared-lint.yml@main

.gitignore

@@ -81,9 +81,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-.python-version
-
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
@@ -131,4 +128,6 @@ dmypy.json
 .DS_Store
 cov_html
 .vscode/
-.idea/
+.idea/
+
+output/

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+default_language_version:
+  python: python3.12
+repos:
+  - repo: local
+    hooks:
+      - id: black-apply
+        name: black-apply
+        entry: pipenv run black
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: mypy
+        name: mypy
+        entry: pipenv run mypy
+        language: system
+        pass_filenames: true
+        types: ["python"]
+        exclude: "tests/"
+      - id: ruff-apply
+        name: ruff-apply
+        entry: pipenv run ruff check --fix
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: safety
+        name: safety
+        entry: pipenv check
+        language: system
+        pass_filenames: false

.python-version

@@ -0,0 +1 @@
+3.12

Dockerfile

@@ -1,20 +1,12 @@
-FROM python:3.9-slim as build
+FROM python:3.12-slim as build
 WORKDIR /app
 COPY . .
-RUN cd /app && python setup.py bdist_wheel
 
-
-FROM python:3.9-slim
-ENV PIP_NO_CACHE_DIR yes
-WORKDIR /app
 RUN pip install --no-cache-dir --upgrade pip pipenv
 
 RUN apt-get update && apt-get upgrade -y && apt-get install -y git
 
 COPY Pipfile* /
-RUN pipenv install --system --clear --deploy
-
-COPY --from=build /app/dist/submitter-*-py3-none-any.whl .
-RUN pip install submitter-*-py3-none-any.whl
+RUN pipenv install
 
-ENTRYPOINT ["submitter"]
+ENTRYPOINT ["pipenv", "run", "submitter"]

Makefile

@@ -1,3 +1,62 @@
+SHELL=/bin/bash
+DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
+
+help: # Preview Makefile commands
+	@awk 'BEGIN { FS = ":.*#"; print "Usage:  make <target>\n\nTargets:" } \
+/^[-_[:alpha:]]+:.?*#/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
+#######################
+# Dependency commands
+#######################
+
+install: # Install Python dependencies
+	pipenv install --dev
+	pipenv run pre-commit install
+
+update: install # Update Python dependencies
+	pipenv clean
+	pipenv update --dev
+
+
+######################
+# Unit test commands
+######################
+
+test: # Run tests and print a coverage report
+	pipenv run coverage run --source=submitter -m pytest -vv
+	pipenv run coverage report -m
+
+coveralls: test # Write coverage data to an LCOV report
+	pipenv run coverage lcov -o ./coverage/lcov.info
+
+
+####################################
+# Code quality and safety commands
+####################################
+
+lint: black mypy ruff safety # Run linters
+
+black: # Run 'black' linter and print a preview of suggested changes
+	pipenv run black --check --diff .
+
+mypy: # Run 'mypy' linter
+	pipenv run mypy .
+
+ruff: # Run 'ruff' linter and print a preview of errors
+	pipenv run ruff check .
+
+safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-date
+	pipenv check
+	pipenv verify
+
+lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
+
+black-apply: # Apply changes with 'black'
+	pipenv run black .
+
+ruff-apply: # Resolve 'fixable errors' with 'ruff'
+	pipenv run ruff check --fix .
+
 ### This is the Terraform-generated header for dspace-submission-service-dev. If  ###
 ###   this is a Lambda repo, uncomment the FUNCTION line below  ###
 ###   and review the other commented lines in the document.     ###
@@ -65,37 +124,4 @@ verify-dspace-connection-prod: # Verify prod app can connect to DSpace
 #   Not yet deployed in production
 
 # verify-dspace-connection-prod: # Verify prod app can connect to DSpace
-# 	Not yet deployed in production
-
-### Dependency commands ###
-install: ## Install script and dependencies
-	pipenv install --dev
-
-update: install ## Update all Python dependencies
-	pipenv clean
-	pipenv update --dev
-
-
-### Testing commands ###
-test: ## Run tests and print a coverage report
-	pipenv run coverage run --source=submitter -m pytest -vv
-	pipenv run coverage report -m
-
-coveralls: test
-	pipenv run coverage lcov -o ./coverage/lcov.info
-
-
-### Linting commands ###
-lint: bandit black flake8 isort ## Lint the repo
-
-bandit:
-	pipenv run bandit -r submitter
-
-black:
-	pipenv run black --check --diff .
-
-flake8:
-	pipenv run flake8 .
-
-isort:
-	pipenv run isort . --diff
+# 	Not yet deployed in production

Pipfile

@@ -11,10 +11,7 @@ sentry-sdk = "*"
 smart-open = "*"
 
 [dev-packages]
-flake8 = "*"
 black = "*"
-isort = "*"
-bandit = "*"
 moto = {extras = ["s3", "server", "sqs"], version = "*"}
 pytest = "*"
 coveralls = "*"
@@ -23,9 +20,15 @@ pytest-cov = "*"
 pytest-env = "*"
 freezegun = "*"
 coverage = "*"
+ruff = "*"
+safety = "*"
+pre-commit = "*"
+mypy = "*"
+types-requests = "*"
+boto3-stubs = {extras = ["essential"], version = "*"}
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"
 
 [scripts]
 submitter = "python -c \"from submitter.cli import main; main()\""