Merge pull request #312 from 4dn-dcic/python-3.12-aftermath

Added numpy ^1.26.4 in pyproject.toml

CHANGELOG.rst

@@ -6,6 +6,20 @@ dcicutils
 Change Log
 ----------
 
+8.13.3
+======
+* N.B. Accidentially tagged/pushed 8.13.1 -> PLEASE IGNORE VERSION: 8.13.1 (subsequently yanked).
+  And then to correct (while no permission to delete above) pushed unofficial 8.13.2.
+* Fallout from Python 3.12 support.
+  - Though dcicutils is not dependent on numpy, elasticsearch tries to import it,
+    and if it is installed and if it is a version greater than 1.x, we get this error:
+    AttributeError: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.
+    So added a hack in  hack_for_elasticsearch_numpy_usage.py for this specific case;
+    to be imported before we import elasticsearch modules.
+* Added/updated scripts from submitr: view_portal_object.py and update_portal_object.py
+  for dev/troubleshooting purposes.
+
+
 8.13.0
 ======
 * Updates related to Python 3.12.

Makefile

@@ -18,61 +18,69 @@ build:  # builds
 test:  # runs default tests, which are the unit tests
 	make test-units
 	make test-static
+	make test-last
 
 test-for-ga:
 	poetry run flake8 dcicutils
 	poetry run flake8 test --exclude=data_files
 	make test-units-with-coverage
+	make test-last
+
+test-last:
+	poetry run pytest -vv -m "last"
 
 retest:  # runs only failed tests from the last test run. (if no failures, it seems to run all?? -kmp 17-Dec-2020)
-	poetry run pytest -vv -r w --last-failed
+	poetry run pytest -vv -r w --last-failed -m "not last"
 
 test-all:  # you have to be really brave to want this. a lot of things will err
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w
+	poetry run pytest -vv -r w -m "not last"
+	make test-last
 	@git log -1 --decorate | head -1
 	@date
 
 test-most:  # leaves out things that will probably err but runs unit tests and both kinds of integrations
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w -m "not static and not beanstalk_failure and not direct_es_query"
+	poetry run pytest -vv -r w -m "not static and not beanstalk_failure and not direct_es_query and not last"
 	@git log -1 --decorate | head -1
 	@date
 
 test-units-with-coverage:
 	@git log -1 --decorate | head -1
 	@date
-	poetry run coverage run --source dcicutils -m pytest -vv -r w -m "not static and not integratedx and not beanstalk_failure and not direct_es_query"
+	poetry run coverage run --source dcicutils -m pytest -vv -r w -m "not static and not integratedx and not beanstalk_failure and not direct_es_query and not last"
+	make test-last
 	@git log -1 --decorate | head -1
 	@date
 
 test-units:  # runs unit tests (and integration tests not backed by a unit test)
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w -m "not static and not integratedx and not beanstalk_failure and not direct_es_query"
+	poetry run pytest -vv -r w -m "not static and not integratedx and not beanstalk_failure and not direct_es_query and not last"
+	make test-last
 	@git log -1 --decorate | head -1
 	@date
 
 test-integrations:  # runs integration tests
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w -m "not static and (integrated or integratedx) and not beanstalk_failure and not direct_es_query"
+	poetry run pytest -vv -r w -m "not static and (integrated or integratedx) and not beanstalk_failure and not direct_es_query and not last"
 	@git log -1 --decorate | head -1
 	@date
 
 test-direct-es-query:  # must be called inside VPC (e.g., from foursight after cloning repo, setting up venv, etc)
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w -m "direct_es_query"
+	poetry run pytest -vv -r w -m "direct_es_query and not last"
 	@git log -1 --decorate | head -1
 	@date
 
 test-static:
 	@git log -1 --decorate | head -1
 	@date
-	poetry run pytest -vv -r w -m "static"
+	poetry run pytest -vv -r w -m "static and not last"
 	poetry run flake8 dcicutils
 	poetry run flake8 test --exclude=data_files
 	@git log -1 --decorate | head -1

dcicutils/es_utils.py

@@ -1,6 +1,7 @@
 import logging
 import boto3
 from .misc_utils import PRINT
+import dcicutils.hack_for_elasticsearch_numpy_usage  # noqa
 from elasticsearch import Elasticsearch, RequestsHttpConnection
 from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
 

dcicutils/ff_utils.py

@@ -7,6 +7,7 @@
 import time
 
 from collections import namedtuple
+import dcicutils.hack_for_elasticsearch_numpy_usage  # noqa
 from elasticsearch.exceptions import AuthorizationException
 from typing import Dict, List, Optional
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse

dcicutils/hack_for_elasticsearch_numpy_usage.py

@@ -0,0 +1,10 @@
+# Though dcicutils is not dependent on numpy, elasticsearch pulls it in iff it is installed,
+# and if it is numpy 2.x the numpy.float_ constant has been retired and any reference to it
+# yields an error from numpy (AttributeError: np.float_ was removed in the NumPy 2.0 release.
+# Use np.float64 instead); this reference to numpy.float_ occurs in elasticsearch/serializer.py.
+# and we short-circuit it here by explicitly setting numpy.float_ to numpyh.float64.
+try:
+    import numpy
+    numpy.float_ = numpy.float64
+except Exception:
+    pass