Merge remote-tracking branch 'origin/master' into cfm-linkTo_items_fix

CHANGELOG.rst

@@ -13,6 +13,12 @@ Change Log
 * Bump SPC with conditional for nested object arrays (without corresponding childKeys values)
 * Update SPC to 0.1.68b2
 
+14.0.0
+======
+* July 2023
+* Migrating ingestion (et.al.) code to snovault.
+
+
 13.5.2
 ======
 `PR 707: Somatic CGAP UI V1 <https://github.com/dbmi-bgm/cgap-portal/pull/707>`_

Makefile

@@ -127,8 +127,8 @@ deploy1:  # starts postgres/ES locally and loads inserts, and also starts ingest
 	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load
 
 deploy1a:  # starts postgres/ES locally and loads inserts, but does not start the ingestion engine
-#	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load --no_ingest
-	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load
+	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load --no_ingest
+#	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load
 
 deploy1b:  # starts ingestion engine separately so it can be easily stopped and restarted for debugging in foreground
 	@echo "Starting ingestion listener. Press ^C to exit." && DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` poetry run ingestion-listener development.ini --app-name app
@@ -333,6 +333,12 @@ lint-full:
 lint:
 	@flake8 deploy/ && flake8 src/encoded/
 
+publish:
+	poetry run publish-to-pypi
+
+publish-for-ga:
+	poetry run publish-to-pypi --noconfirm
+
 help:
 	@make info
 

bin/macpoetry-install

@@ -17,9 +17,12 @@ if [[ `uname -o` == "Darwin" && `uname -m` == "arm64" ]] ; then
     # /System/Volumes/Preboot/Cryptexes/OS/.../.pyenv/versions/3.8.13/envs/cg-3.8.13/lib/python3.8/site-packages/tabix.cpython-38-darwin.so
     # (no such file), .../.pyenv/versions/3.8.13/envs/cg-3.8.13/lib/python3.8/site-packages/tabix.cpython-38-darwin.so
     # (mach-o file, but is an incompatible architecture (have x86_64, need arm64))
-    #
     pip uninstall -y pytabix
     pip install pytabix
+    # For some reason installing psycopg2 via poetry gets
+    # the x86_64 so file but via pip install the correct arm64 one.
+    pip uninstall -y psycopg2 psycopg2-binary
+    pip install psycopg2 psycopg2-binary
     echo "Done with post poetry install special installs for Mac M1."
 fi
 

deploy/docker/local/docker_development.ini.template

@@ -8,8 +8,8 @@
 use = config:base.ini#app
 session.secret = %(here)s/session-secret.b64
 env.name = cgap-docker-${USER}-test
-sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres
-elasticsearch.server = search-cgap-unit-testing-6-8-2p5zrlkmxif4bayh5y6kxzlvx4.us-east-1.es.amazonaws.com:443
+sqlalchemy.url = postgresql://postgres:postgres@db:5432/postgres
+elasticsearch.server = search-cgap-unit-testing-opensearch-tcs45cjpwgdzoi7pafr6oewq6u.us-east-1.es.amazonaws.com:443
 elasticsearch.aws_auth = true
 blob_bucket = cgap-unit-testing-blobs
 metadata_bundles_bucket = cgap-unit-testing-metadata-bundles
@@ -27,7 +27,7 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 # this line determines which load function is used in load_data
 # most deployments use: "load_test_data = encoded.loadxl:load_test_data"
-load_test_data = encoded.loadxl:load_${DATA_SET}_data
+load_test_data = snovault.loadxl:load_${DATA_SET}_data
 encoded_version = 100.200.300
 snovault_version = 200.300.400
 utils_version = 300.400.500

deploy/docker/production/cgap_any_alpha.ini

@@ -33,7 +33,7 @@ indexer.namespace = ${ES_NAMESPACE}
 index_server = ${INDEX_SERVER}
 elasticsearch.aws_auth = true
 production = true
-load_test_data = encoded.loadxl:load_${DATA_SET}_data
+load_test_data = snovault.loadxl:load_${DATA_SET}_data
 sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME}
 
 [composite:indexer]

development.ini.template

@@ -22,7 +22,7 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 # this line determines which load function is used in load_data
 # most deployments use: "load_test_data = encoded.loadxl:load_test_data"
-load_test_data = encoded.loadxl:load_local_data
+load_test_data = snovault.loadxl:load_local_data
 encoded_version = 100.200.300
 snovault_version = 200.300.400
 utils_version = 300.400.500

docker-compose.yml.template

@@ -69,3 +69,18 @@ services:
     depends_on:
       - db
       #- es
+
+# TODO: This is not yet fully setup.
+# Need to fix up entrypoint scripts to call make deploy1a for ingester in docker.
+# ingester:
+#   build: .
+#   container_name: ingester
+#   command: "/home/nginx/cgap-portal/entrypoint_ingester.sh"
+#   environment:
+#     # Default to caller's environment
+#     AWS_ACCESS_KEY_ID:
+#     AWS_SECRET_ACCESS_KEY:
+#     GLOBAL_ENV_BUCKET:
+#     Auth0Client:
+#     Auth0Secret:
+#     application_type: "local"  # use local entrypoint