From 35c1cac43f198e64de24b39d188d0ecf3420081b Mon Sep 17 00:00:00 2001 From: andykawabata Date: Tue, 23 Apr 2024 11:33:46 -0400 Subject: [PATCH 1/2] backend maintenance and update readme --- .gitignore | 2 ++ README.md | 3 ++- backend/docker/requirements.in | 11 +++++++---- backend/docker/requirements.txt | 31 ++++++++++++++++--------------- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index df8d6b02..c9bc557d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ backup/* # IDE specific .vscode/ + +venv \ No newline at end of file diff --git a/README.md b/README.md index ff3f4e51..8845e598 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,9 @@ docker-compose run --rm backend ./migrate.sh We use [pip-tools](https://github.com/jazzband/pip-tools) to manage Python dependencies. To change the dependencies: -1. Edit [requirements.in](./backend/docker/requirements.in) to add, remove, or edit a dependency. You only need to put primary dependencies here, that is, the ones explicitly needed by our source code. pip-tools will take care of adding their dependencies. +1. Edit [requirements.in](./backend/docker/requirements.in) file to add, remove, or update dependencies as needed. Include only the primary dependencies—those directly required by our source code—in this file. pip-tools will automatically manage and incorporate any transitive dependencies. For routine maintenance, you may specify both primary and transitive dependencies with pinned versions to ensure consistent updates. 1. Run `docker-compose run --rm backend pip-compile docker/requirements.in` to generate a new [requirements.txt](./backend/docker/requirements.txt). Note that pip-tools uses the existing `requirements.txt` file when building a new one, so that it can maintain existing versions. To upgrade a package to the newest version compatible with the other libraries, just remove it from the existing `requirements.txt` before running pip-compile. + - If you encounter an error while running with docker, a possible workaround is to create a virtual environment within `backend/docker`, install `pip-tools` within that environment, and then run `pip-compile requirements.in` directly from there. This method bypasses network issues that might occur within Docker. 1. Run `docker-compose build backend` to install the updated requirements into the Docker image. ### Custom Environment Variables diff --git a/backend/docker/requirements.in b/backend/docker/requirements.in index 8efe7571..8b9e2805 100644 --- a/backend/docker/requirements.in +++ b/backend/docker/requirements.in @@ -1,7 +1,8 @@ # main dependencies for the backend application +black>=24.3.0 celery -cryptography>=41.0.6 -Django>=4.2.7 +cryptography>=42.0.4 +Django>=4.2.11 django-allauth django-configurations django-extensions @@ -12,10 +13,11 @@ django-formtools django-rest-swagger django-webpack-loader djangorestframework-jwt +idna>=3.7 jinja2>=3.1.3 openpyxl pandas -Pillow +Pillow>=10.3.0 pyodbc redis requests @@ -23,7 +25,8 @@ scikit-learn sentence_transformers==2.2.1 # starting at 2.3.0 sentence_transformers requires upgrading torch to 1.11.0 - which appears to be a significant change statsmodels sqlalchemy -transformers>=4.36.0 +sqlparse>=0.5.0 +transformers>=4.38.0 xlrd # dependencies for code formatters, dependency management, testing, etc. diff --git a/backend/docker/requirements.txt b/backend/docker/requirements.txt index 6f723d25..143eb10e 100644 --- a/backend/docker/requirements.txt +++ b/backend/docker/requirements.txt @@ -16,7 +16,7 @@ backports-zoneinfo==0.2.1 # via django billiard==3.6.4.0 # via celery -black==21.11b1 +black==24.4.0 # via -r requirements.in build==1.0.3 # via pip-tools @@ -55,7 +55,7 @@ coreschema==0.0.4 # via coreapi coverage==6.1.2 # via -r requirements.in -cryptography==42.0.2 +cryptography==42.0.5 # via # -r requirements.in # pyjwt @@ -65,7 +65,7 @@ distlib==0.3.3 # via virtualenv dj-rest-auth==3.0.0 # via -r requirements.in -django==4.2.9 +django==4.2.11 # via # -r requirements.in # dj-rest-auth @@ -122,8 +122,10 @@ huggingface-hub==0.20.3 # transformers identify==2.3.6 # via pre-commit -idna==3.3 - # via requests +idna==3.7 + # via + # -r requirements.in + # requests importlib-metadata==7.0.1 # via build iniconfig==1.1.1 @@ -168,8 +170,9 @@ openapi-codec==1.3.2 # via django-rest-swagger openpyxl==3.0.9 # via -r requirements.in -packaging==21.2 +packaging==24.0 # via + # black # build # huggingface-hub # pytest @@ -182,7 +185,7 @@ pathspec==0.9.0 # via black patsy==0.5.2 # via statsmodels -pillow==10.2.0 +pillow==10.3.0 # via # -r requirements.in # torchvision @@ -210,11 +213,8 @@ pyjwt[crypto]==1.7.1 # via # django-allauth # djangorestframework-jwt - # pyjwt pyodbc==4.0.35 # via -r requirements.in -pyparsing==2.4.7 - # via packaging pyproject-hooks==1.0.0 # via build pytest==7.3.1 @@ -239,7 +239,6 @@ redis==5.0.1 # via -r requirements.in regex==2021.11.10 # via - # black # nltk # transformers requests==2.28.2 @@ -277,8 +276,10 @@ six==1.16.0 # virtualenv sqlalchemy==1.4.28 # via -r requirements.in -sqlparse==0.4.2 - # via django +sqlparse==0.5.0 + # via + # -r requirements.in + # django statsmodels==0.13.1 # via -r requirements.in threadpoolctl==3.0.0 @@ -306,11 +307,11 @@ tqdm==4.62.3 # nltk # sentence-transformers # transformers -transformers==4.37.2 +transformers==4.39.3 # via # -r requirements.in # sentence-transformers -typing-extensions==4.0.0 +typing-extensions==4.11.0 # via # black # huggingface-hub From dd7ea35ad8e0f38254dfcfddc23721fccd4ddab8 Mon Sep 17 00:00:00 2001 From: Caroline Kery Date: Tue, 21 May 2024 13:19:47 -0400 Subject: [PATCH 2/2] making small changes to account for running the upgraded black on the repo and making sure pytest can run --- backend/django/core/utils/utils_external_db.py | 6 +++--- backend/django/core/utils/utils_form.py | 1 - backend/django/core/views/api_annotate.py | 12 ++++++------ backend/django/run_tests.sh | 2 +- backend/django/smart/settings.py | 1 + backend/django/smart/urls.py | 1 + 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/django/core/utils/utils_external_db.py b/backend/django/core/utils/utils_external_db.py index 4483dcfd..b2f09099 100644 --- a/backend/django/core/utils/utils_external_db.py +++ b/backend/django/core/utils/utils_external_db.py @@ -208,9 +208,9 @@ def export_table(project_pk, response): external_db.export_schema, external_db.export_table_name, ): - response[ - "success_message" - ] = "Table exists. Dropping and replacing with new output data." + response["success_message"] = ( + "Table exists. Dropping and replacing with new output data." + ) # drop the table and then replace the data data.to_sql( diff --git a/backend/django/core/utils/utils_form.py b/backend/django/core/utils/utils_form.py index e5b1fd00..2f184e58 100644 --- a/backend/django/core/utils/utils_form.py +++ b/backend/django/core/utils/utils_form.py @@ -1,5 +1,4 @@ import numpy as np -import pandas as pd from django.core.exceptions import ValidationError from core.utils.util import md5_hash diff --git a/backend/django/core/views/api_annotate.py b/backend/django/core/views/api_annotate.py index e15671cf..1e72f120 100644 --- a/backend/django/core/views/api_annotate.py +++ b/backend/django/core/views/api_annotate.py @@ -220,9 +220,9 @@ def verify_label(request, data_pk): response = {} # if the coder has been un-assigned from the data if not DataLabel.objects.filter(data=data).exists(): - response[ - "error" - ] = "ERROR: This data has no label to verify. Something must have gone wrong." + response["error"] = ( + "ERROR: This data has no label to verify. Something must have gone wrong." + ) return Response(response) elif DataLabel.objects.filter(data=data).count() > 1: response["error"] = ( @@ -1089,9 +1089,9 @@ def get_label_history(request, project_pk): if len(irr_data_df) > 0: irr_data_df["edit"] = "no" irr_data_df["label"] = irr_data_df["labelID"].apply(lambda x: label_dict[x]) - irr_data_df[ - "verified" - ] = "N/A (IRR)" # Technically resolved IRR is verified but perhaps not this user's specific label so just NA + irr_data_df["verified"] = ( + "N/A (IRR)" # Technically resolved IRR is verified but perhaps not this user's specific label so just NA + ) irr_data_df["verified_by"] = None irr_data_df["pre_loaded"] = "No" # IRR only looks at unlabeled data diff --git a/backend/django/run_tests.sh b/backend/django/run_tests.sh index c0bcee7d..aef52301 100755 --- a/backend/django/run_tests.sh +++ b/backend/django/run_tests.sh @@ -20,5 +20,5 @@ else echo "Flake8 Good. Running Tests..." fi -coverage run -m py.test "$@" +coverage run -m pytest "$@" coverage report -m --skip-covered diff --git a/backend/django/smart/settings.py b/backend/django/smart/settings.py index b7124809..cdc262c1 100644 --- a/backend/django/smart/settings.py +++ b/backend/django/smart/settings.py @@ -8,6 +8,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ + import os import redis diff --git a/backend/django/smart/urls.py b/backend/django/smart/urls.py index c2752d2e..8ec93efb 100644 --- a/backend/django/smart/urls.py +++ b/backend/django/smart/urls.py @@ -13,6 +13,7 @@ 1. Import the include() function: from django.urls import re_path, include 2. Add a URL to urlpatterns: re_path(r'^blog/', include('blog.urls')) """ + from django.contrib import admin from django.http import HttpResponseRedirect from django.urls import include, re_path