Merge branch 'main' of https://github.com/sennetconsortium/entity-api

.github/workflows/tests.yaml

@@ -12,10 +12,21 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: [3.9]
+
+    services:
+      neo4j:
+        image: neo4j:5.20.0-ubi8
+        ports:
+          - 7474:7474
+          - 7687:7687
+        env:
+          NEO4J_AUTH: none
+          NEO4JLABS_PLUGINS: '["apoc"]'
+
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -29,7 +40,11 @@ jobs:
           pip install -r src/requirements.dev.txt
       - name: Setup test config file
         run: |
-          cp test/config/app.cfg src/instance/app.cfg
+          cp test/config/app.test.cfg src/instance/app.cfg
       - name: Test with pytest
+        env:
+          UBKG_SERVER: ${{ secrets.UBKG_SERVER }}
+          UBKG_ENDPOINT_VALUESET: ${{ secrets.UBKG_ENDPOINT_VALUESET }}
+          UBKG_CODES: ${{ secrets.UBKG_CODES }}
         run: |
           pytest -W ignore::DeprecationWarning 

README.md

@@ -46,6 +46,5 @@ cd docker
 The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and committing the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents.  The documentation is associated with this github account: api-developers@sennetconsortium.org.
 
 ## Testing
-Install the development dependencies using `pip install -r src/requirements.dev.txt`.
 
-Execute `pytest` in the root directory to run all unit tests. `pytest -W ignore::DeprecationWarning` will execute all unit tests without deprecation warnings.
+Install the development dependencies using `pip install -r src/requirements.dev.txt`. Install Docker and ensure it is running. Run `./run_tests.sh` at the root of the project. This test script will create a temporary Neo4J database using Docker for integration tests.

run_tests.sh

@@ -0,0 +1,43 @@
+# check if the neo4j-test docker container is running and stop it
+if [ "$(docker ps -q -f name=neo4j-test)" ]; then
+    echo "Stopping the existing neo4j-test container"
+    docker stop neo4j-test > /dev/null
+fi
+
+# check if the neo4j-test docker container exists and remove it
+if [ "$(docker ps -aq -f name=neo4j-test)" ]; then
+    echo "Removing the existing neo4j-test container"
+    docker rm neo4j-test > /dev/null
+fi
+
+# create a new neo4j-test docker container
+echo "Creating a new neo4j-test container"
+docker run -d \
+    --name neo4j-test \
+    -p 7474:7474 \
+    -p 7687:7687 \
+    -e NEO4J_AUTH=none \
+    -e NEO4JLABS_PLUGINS=\[\"apoc\"\] \
+    neo4j:5.20.0-ubi8
+
+# Read values from config file and set them as environment variables
+UBKG_SERVER=$(awk -F ' = ' '/UBKG_SERVER/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//")
+UBKG_ENDPOINT_VALUESET=$(awk -F ' = ' '/UBKG_ENDPOINT_VALUESET/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//")
+UBKG_CODES=$(awk -F ' = ' '/UBKG_CODES/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//")
+
+# Set the test config file and backup the original config file
+mv src/instance/app.cfg src/instance/app.cfg.bak
+cp test/config/app.test.cfg src/instance/app.cfg
+
+echo "Running tests"
+UBKG_SERVER=$UBKG_SERVER \
+UBKG_ENDPOINT_VALUESET=$UBKG_ENDPOINT_VALUESET \
+UBKG_CODES=$UBKG_CODES \
+pytest -W ignore::DeprecationWarning
+
+# Restore the original config file
+mv src/instance/app.cfg.bak src/instance/app.cfg
+
+echo "Stopping and removing the neo4j-test container"
+docker stop neo4j-test > /dev/null
+docker rm --volumes neo4j-test > /dev/null

src/app.py

@@ -5517,6 +5517,9 @@ def verify_ubkg_properties(json_data_dict):
     if 'intended_organ' in json_data_dict:
         compare_property_against_ubkg(ORGAN_TYPES, json_data_dict, 'intended_organ')
 
+    if 'intended_source_type' in json_data_dict:
+        compare_property_against_ubkg(SOURCE_TYPES, json_data_dict, 'intended_source_type')
+
 def compare_property_list_against_ubkg(ubkg_dict, json_data_dict, field):
     good_fields = []
     passes_ubkg_validation = True

src/instance/app.cfg.example

@@ -70,4 +70,4 @@ UBKG_SERVER =
 UBKG_ENDPOINT_VALUESET =
 UBKG_CODES =
 
-MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU']
+MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU']

src/requirements.txt

@@ -11,7 +11,7 @@ nested-lookup==0.2.22
 
 # The commons package requires requests>=2.22.0 and PyYAML>=5.3.1
 requests==2.32.3
-PyYAML==5.4.1
+PyYAML==6.0
 
 # Pinning specific version of neo4j
 neo4j==5.22.0
@@ -21,6 +21,6 @@ neo4j==5.22.0
 # Default is main branch specified in docker-compose.development.yml if not set
 # git+https://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons
 hubmap-commons==2.1.18
-atlas-consortia-commons==1.0.10
+atlas-consortia-commons==1.1.0
 
 deepdiff~=6.6.0

src/schema/provenance_schema.yaml

@@ -355,8 +355,17 @@ ENTITIES:
     before_entity_create_validator: validate_application_header_before_entity_create
     excluded_properties_from_public_response:
       - lab_dataset_id
+      - direct_ancestors:
+          - lab_tissue_sample_id
+          - source:
+              - lab_source_id
+          - origin_samples:
+              - lab_tissue_sample_id
       - sources:
           - lab_source_id
+      - metadata:
+          - lab_id
+          - slide_id
       - ingest_metadata:
           - metadata:
               - lab_id
@@ -1290,6 +1299,11 @@ ENTITIES:
         type: string
         description: 'The organ code representing the organ type that the data contained in the upload will be registered/associated with.'
         required_on_create: true
+      intended_source_type:
+        type: string
+        description: 'The source type that the data contained in the upload will be registered/associated with.'
+        required_on_create: true
+
 
   ############################################# EPICollection #############################################
   Epicollection:

src/schema/schema_manager.py

@@ -343,7 +343,11 @@ def delete_nested_field(data, nested_path):
                     if isinstance(value, list):
                         for nested_field in value:
                             if isinstance(nested_field, dict):
-                                delete_nested_field(data[key], nested_field)
+                                if isinstance(data[key], list):
+                                    for item in data[key]:
+                                        delete_nested_field(item, nested_field)
+                                else:
+                                    delete_nested_field(data[key], nested_field)
 
                             elif isinstance(data[key], list):
                                 for item in data[key]:

test/config/app.cfg → test/config/app.test.cfg

@@ -5,14 +5,14 @@ READ_ONLY_MODE = False
 SCHEMA_YAML_FILE = './schema/provenance_schema.yaml'
 
 # Globus App ID and secret
-APP_CLIENT_ID = 'c4018852'
-APP_CLIENT_SECRET = 'supersecret'
+APP_CLIENT_ID = ''
+APP_CLIENT_SECRET = ''
 
 # Neo4j connection (default value used for docker localhost deployment)
 # Point to remote neo4j for dev/test/stage/prod deployment
-NEO4J_URI = 'bolt://hubmap-neo4j-localhost:7687'
+NEO4J_URI = 'bolt://localhost:7687'
 NEO4J_USERNAME = 'neo4j'
-NEO4J_PASSWORD = '123'
+NEO4J_PASSWORD = None
 
 # Set MEMCACHED_MODE to False to disable the caching for local development
 MEMCACHED_MODE = False
@@ -46,9 +46,9 @@ GLOBUS_APP_BASE_URL = 'https://app.globus.org'
 
 # Below configurations are for DOI redirection
 # UUIDs of the Globus endpoints
-GLOBUS_PUBLIC_ENDPOINT_UUID = '6be4ac4f-7b63-4640-9a19-3eb6d6e779d6'
-GLOBUS_CONSORTIUM_ENDPOINT_UUID = '4be0eae8-ce38-41f3-bede-86d364f72201'
-GLOBUS_PROTECTED_ENDPOINT_UUID = '6be4ac4f-7b63-4640-9a19-3eb6d6e779d6'
+GLOBUS_PUBLIC_ENDPOINT_UUID = ''
+GLOBUS_CONSORTIUM_ENDPOINT_UUID = ''
+GLOBUS_PROTECTED_ENDPOINT_UUID = ''
 
 # Sub directories under the base data/globus directory where different access levels of data sits
 PROTECTED_DATA_SUBDIR = 'private'
@@ -63,8 +63,9 @@ DOI_REDIRECT_URL = 'https://data.sennetconsortium.org/<entity_type>?uuid=<identi
 #URL to tsv file that holds the redirect url information for DOI redirects
 REDIRECTION_INFO_URL = 'https://raw.githubusercontent.com/hubmapconsortium/ccf-releases/main/reference-entity-ids.tsv'
 
-UBKG_SERVER = 'http://ubkg:8080'
-UBKG_ENDPOINT_VALUESET = ''
-UBKG_CODES = ''
+import os
+UBKG_SERVER = os.getenv('UBKG_SERVER')
+UBKG_ENDPOINT_VALUESET = os.getenv('UBKG_ENDPOINT_VALUESET')
+UBKG_CODES = os.getenv('UBKG_CODES')
 
-MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU']
+MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU']

test/conftest.py

@@ -0,0 +1 @@
+pytest_plugins = ["test.helpers.auth", "test.helpers.database", "test.helpers.request"]