Upgrade to Python 3.11

.github/workflows/main.yml

@@ -17,17 +17,17 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: Test Suite for fourfront (Python 3.8, Node 18)
+    name: Test Suite for fourfront (Python 3.11, Node 18)
 
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Build matrix
     strategy:
       matrix:
         test_type: ['UNIT', 'NPM', 'Docker']
         # We are really not set up for these next two to be multiplicative, so be careful adding more.
-        python_version: ['3.8']
+        python_version: ['3.11']
         node_version: ['18']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -125,8 +125,8 @@ jobs:
           poetry run wipe-test-indices $TEST_JOB_ID search-fourfront-testing-opensearch-kqm7pliix4wgiu4druk2indorq.us-east-1.es.amazonaws.com:443
 
       - name: Docker Build
-        if: ${{ matrix.test_type == 'Docker' && matrix.node_version == '18' && matrix.python_version == '3.8' }}
+        if: ${{ matrix.test_type == 'Docker' && matrix.node_version == '18' && matrix.python_version == '3.11' }}
         run: |
-          # The docker_development.ini has node 18 and Python 3.8 wired into it.
+          # The docker_development.ini has node 18 and Python 3.11 wired into it.
           touch deploy/docker/local/docker_development.ini  # cheap substitute for prepare-docker to make ignored file
           docker build .

CHANGELOG.rst

@@ -6,6 +6,17 @@ fourfront
 Change Log
 ----------
 
+7.0.0
+=====
+
+* Upgrade to Python 3.11.
+* Changed pyyaml version to ^6.0.1.
+* Removed types/access_key.py and schemas/access_key.json as the ones in snovault are sufficient.
+* Added generate-local-access-key script (from snovault) to pyproject.toml;
+  orignally created for smaht-portal since early in development no way to
+  create an access-key normally using the UI; but generally useful/convenient.
+
+
 6.4.6
 =====
 

Dockerfile

@@ -1,8 +1,10 @@
 # Fourfront (Production) Dockerfile
 # Based off of the cgap-portal Dockerfile
 
-# Debian Buster with Python 3.8.13
-FROM python:3.8.13-slim-buster
+# Bullseye with Python 3.11.5
+# 2023-09-28: Update docker image to a Python 3.11 version;
+# this was previously: FROM python:3.8.13-slim-buster
+FROM python:3.11.5-slim-bullseye
 
 MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu"
 
@@ -33,7 +35,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 # Note that the ordering of these operations is intentional to minimize package footprint
 WORKDIR /home/nginx/.nvm
 ENV NVM_DIR=/home/nginx/.nvm
-COPY deploy/docker/production/install_nginx.sh /install_nginx.sh
+COPY deploy/docker/production/install_nginx_bullseye.sh /install_nginx.sh
 
 # Temporarily replacing
 #     curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/venv python - && \

deploy/docker/production/install_nginx_bullseye.sh

@@ -0,0 +1,76 @@
+# Copied from: https://github.com/nginxinc/docker-nginx/blob/master/mainline/debian/Dockerfile March 3 2022 -Will
+
+export NGINX_VERSION=1.21.6
+export NJS_VERSION=0.7.2
+export PKG_RELEASE=1~bullseye
+
+set -x \
+# create nginx user/group first, to be consistent throughout docker variants
+    addgroup --system --gid 121 nginx \
+    && adduser --system --disabled-login --ingroup nginx --no-create-home --home /nonexistent --gecos "nginx user" --shell /bin/false --uid 121 nginx \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y gnupg2 ca-certificates \
+    && \
+    NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+    found=''; \
+    for server in \
+        hkp://keyserver.ubuntu.com:80 \
+        pgp.mit.edu \
+    ; do \
+        echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+        apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+    done; \
+    test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+    apt-get remove --purge --auto-remove -y gnupg2 && rm -rf /var/lib/apt/lists/* \
+    && dpkgArch="$(dpkg --print-architecture)" \
+    && nginxPackages=" \
+        nginx=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-njs=${NGINX_VERSION}+${NJS_VERSION}-${PKG_RELEASE} \
+    " \
+    && case "$dpkgArch" in \
+        amd64|arm64) \
+            echo "deb https://nginx.org/packages/mainline/debian/ bullseye nginx" >> /etc/apt/sources.list.d/nginx.list \
+            && apt-get update \
+            ;; \
+        *) \
+            echo "deb-src https://nginx.org/packages/mainline/debian/ bullseye nginx" >> /etc/apt/sources.list.d/nginx.list \
+            \
+            && tempDir="$(mktemp -d)" \
+            && chmod 777 "$tempDir" \
+            \
+            && savedAptMark="$(apt-mark showmanual)" \
+            \
+            && apt-get update \
+            && apt-get build-dep -y $nginxPackages \
+            && ( \
+                cd "$tempDir" \
+                && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \
+                    apt-get source --compile $nginxPackages \
+            ) \
+            \
+            && apt-mark showmanual | xargs apt-mark auto > /dev/null \
+            && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \
+            \
+            && ls -lAFh "$tempDir" \
+            && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \
+            && grep '^Package: ' "$tempDir/Packages" \
+            && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \
+            && apt-get -o Acquire::GzipIndexes=false update \
+            ;; \
+    esac \
+    \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+                        $nginxPackages \
+                        gettext-base \
+                        curl \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \
+    \
+    && if [ -n "$tempDir" ]; then \
+        apt-get purge -y --auto-remove \
+        && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \
+    fi \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "fourfront".
 name = "encoded"
-version = "6.4.6"
+version = "7.0.0"
 description = "4DN-DCIC Fourfront"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"
@@ -31,24 +31,26 @@ classifiers = [
     # Specify the Python versions you support here. In particular, ensure
     # that you indicate whether you support Python 2, Python 3 or both.
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.7',
-    'Programming Language :: Python :: 3.8'
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11'
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.10"
-awscli = ">=1.25.36"
-boto3 = "^1.26.133"
-botocore = "^1.27.36"
+python = ">=3.8.1,<3.12"
+awscli = ">=1.29.62"
+boto3 = "^1.28.62"
+botocore = "^1.31.62"
 certifi = ">=2021.5.30"
-chardet = "3.0.4"
+chardet = "^5.2.0"
 colorama = "0.3.3"
 # At least on Mac M1 if cryptography package is later than 39.x.x (i.e. 40.x.x and up)
 # we get odd 'pyo3_runtime.PanicException: Python API call failed' error on import
 # of cryptography.hazmat.bindings._rust in cryptography package. 2023-04-21.
 cryptography = "39.0.2"
-dcicsnovault = "^10.0.4"
-dcicutils = "^7.5.0"
+dcicsnovault = "^11.0.0"
+dcicutils = "^8.0.0"
 elasticsearch = "7.13.4"
 elasticsearch-dsl = "^7.0.0"  # TODO: port code from cgap-portal to get rid of uses
 execnet = "1.4.1"
@@ -81,7 +83,7 @@ python-dateutil = "^2.8.2"
 # python-magic is presently pinned to 0.4.15 in lockstep with dcicsnovault's requirements. See explanation there.
 python_magic = ">=0.4.24,<1"
 pytz = ">=2021.3"
-PyYAML = "5.3.1"
+PyYAML = "^6.0.1"
 rdflib = "^4.2.2"
 rdflib-jsonld = ">=0.5.0,<1.0.0"
 requests = "^2.26.0"
@@ -121,8 +123,8 @@ wheel = "0.40.0"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-boto3-stubs = "^1.24.36"
-botocore-stubs = "^1.27.36"
+boto3-stubs = "^1.28.62"
+botocore-stubs = "^1.31.62"
 coverage = ">=6.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=3.3.1"
@@ -186,9 +188,9 @@ dis2pheno = "encoded.commands.parse_hpoa:main"
 es-index-data = "snovault.commands.es_index_data:main"
 export-data = "encoded.commands.export_data:main"
 extract-test-data = "encoded.commands.extract_test_data:main"
+generate-local-access-key = "snovault.commands.generate_local_access_key:main"
 generate-ontology = "encoded.commands.generate_ontology:main"
 import-data = "encoded.commands.import_data:main"
-#ingestion-listener = "encoded.ingestion.ingestion_listener:main"
 ingestion-listener = "snovault.ingestion.ingestion_listener:main"
 jsonld-rdf = "encoded.commands.jsonld_rdf:main"
 load-access-keys = "encoded.commands.load_access_keys:main"
@@ -215,7 +217,6 @@ main = "encoded:main"
 
 [paste.composite_factory]
 indexer = "snovault.elasticsearch.es_index_listener:composite"
-#ingester = "encoded.ingestion_listener:composite"
 ingester = "snovault.ingestion.ingestion_listener:composite"
 
 [paste.filter_app_factory]

src/encoded/commands/extract_test_data.py

@@ -5,7 +5,7 @@
 import sys
 import traceback
 
-from snovault.loadxl import ORDER, read_single_sheet
+from snovault.loadxl import loadxl_order, read_single_sheet
 from lorem_text import lorem
 
 
@@ -122,7 +122,7 @@ def anon_pipeline():
 
 
 def run(pipeline, inpath, outpath):
-    for item_type in ORDER:
+    for item_type in loadxl_order():
         source = read_single_sheet(inpath, item_type)
         fieldnames = [k for k in source.fieldnames if ':ignore' not in k]
         with open(os.path.join(outpath, item_type + '.tsv'), 'wb') as out:

src/encoded/tests/test_schemas.py

@@ -170,7 +170,7 @@ def test_linkTo_saves_uuid(root, submitter, lab):
 
 
 def test_mixinProperties():
-    schema = load_schema('encoded:schemas/access_key.json')
+    schema = load_schema('snovault:schemas/access_key.json')
     assert schema['properties']['uuid']['type'] == 'string'