diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index aa2d1a79..37af144d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,10 +12,21 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: python-version: [3.9] + + services: + neo4j: + image: neo4j:5.20.0-ubi8 + ports: + - 7474:7474 + - 7687:7687 + env: + NEO4J_AUTH: none + NEO4JLABS_PLUGINS: '["apoc"]' + steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -29,7 +40,11 @@ jobs: pip install -r src/requirements.dev.txt - name: Setup test config file run: | - cp test/config/app.cfg src/instance/app.cfg + cp test/config/app.test.cfg src/instance/app.cfg - name: Test with pytest + env: + UBKG_SERVER: ${{ secrets.UBKG_SERVER }} + UBKG_ENDPOINT_VALUESET: ${{ secrets.UBKG_ENDPOINT_VALUESET }} + UBKG_CODES: ${{ secrets.UBKG_CODES }} run: | pytest -W ignore::DeprecationWarning diff --git a/README.md b/README.md index 05e8da25..0e3c9398 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,5 @@ cd docker The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and committing the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents. The documentation is associated with this github account: api-developers@sennetconsortium.org. ## Testing -Install the development dependencies using `pip install -r src/requirements.dev.txt`. -Execute `pytest` in the root directory to run all unit tests. `pytest -W ignore::DeprecationWarning` will execute all unit tests without deprecation warnings. \ No newline at end of file +Install the development dependencies using `pip install -r src/requirements.dev.txt`. Install Docker and ensure it is running. Run `./run_tests.sh` at the root of the project. This test script will create a temporary Neo4J database using Docker for integration tests. diff --git a/docker/test/neo4j/neo4j.conf b/docker/test/neo4j/neo4j.conf new file mode 100644 index 00000000..4af361cc --- /dev/null +++ b/docker/test/neo4j/neo4j.conf @@ -0,0 +1,344 @@ +#***************************************************************** +# Neo4j configuration +# +# For more details and a complete list of settings, please see +# https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/ +#***************************************************************** + +# The name of the default database +dbms.default_database=sennet + +# Paths of directories in the installation. +#server.directories.data=data +#server.directories.plugins=plugins +#server.directories.logs=logs +#server.directories.lib=lib +#server.directories.run=run +#server.directories.licenses=licenses +#server.directories.transaction.logs.root=data/transactions + +# This setting constrains all `LOAD CSV` import files to be under the `import` directory. Remove or comment it out to +# allow files to be loaded from anywhere in the filesystem; this introduces possible security problems. See the +# `LOAD CSV` section of the manual for details. +server.directories.import=import + +# Whether requests to Neo4j are authenticated. +# To disable authentication, uncomment this line +dbms.security.auth_enabled=false + +# Anonymous usage data reporting +# To disable, uncomment this line +#dbms.usage_report.enabled=false + +#******************************************************************** +# Memory Settings +#******************************************************************** +# +# Memory settings are specified kibibytes with the 'k' suffix, mebibytes with +# 'm' and gibibytes with 'g'. +# If Neo4j is running on a dedicated server, then it is generally recommended +# to leave about 2-4 gigabytes for the operating system, give the JVM enough +# heap to hold all your transaction state and query context, and then leave the +# rest for the page cache. + +# Java Heap Size: by default the Java heap size is dynamically calculated based +# on available system resources. Uncomment these lines to set specific initial +# and maximum heap size. +#server.memory.heap.initial_size=512m +#server.memory.heap.max_size=512m + +# The amount of memory to use for mapping the store files. +# The default page cache memory assumes the machine is dedicated to running +# Neo4j, and is heuristically set to 50% of RAM minus the Java heap size. +#server.memory.pagecache.size=10g + +# Limit the amount of memory that all of the running transaction can consume. +# The default value is 70% of the heap size limit. +#dbms.memory.transaction.total.max=256m + +# Limit the amount of memory that a single transaction can consume. +# By default there is no limit. +#db.memory.transaction.max=16m + +#***************************************************************** +# Network connector configuration +#***************************************************************** + +# With default configuration Neo4j only accepts local connections. +# To accept non-local connections, uncomment this line: +server.default_listen_address=0.0.0.0 + +# You can also choose a specific network interface, and configure a non-default +# port for each connector, by setting their individual listen_address. + +# The address at which this server can be reached by its clients. This may be the server's IP address or DNS name, or +# it may be the address of a reverse proxy which sits in front of the server. This setting may be overridden for +# individual connectors below. +#server.default_advertised_address=localhost + +# You can also choose a specific advertised hostname or IP address, and +# configure an advertised port for each connector, by setting their +# individual advertised_address. + +# By default, encryption is turned off. +# To turn on encryption, an ssl policy for the connector needs to be configured +# Read more in SSL policy section in this file for how to define a SSL policy. + +# Bolt connector +server.bolt.enabled=true +#server.bolt.tls_level=DISABLED +#server.bolt.listen_address=:7687 +#server.bolt.advertised_address=:7687 + +# HTTP Connector. There can be zero or one HTTP connectors. +server.http.enabled=true +#server.http.listen_address=:7474 +#server.http.advertised_address=:7474 + +# HTTPS Connector. There can be zero or one HTTPS connectors. +server.https.enabled=false +#server.https.listen_address=:7473 +#server.https.advertised_address=:7473 + +# Number of Neo4j worker threads. +#server.threads.worker_count= + +#***************************************************************** +# SSL policy configuration +#***************************************************************** + +# Each policy is configured under a separate namespace, e.g. +# dbms.ssl.policy..* +# can be any of 'bolt', 'https', 'cluster' or 'backup' +# +# The scope is the name of the component where the policy will be used +# Each component where the use of an ssl policy is desired needs to declare at least one setting of the policy. +# Allowable values are 'bolt', 'https', 'cluster' or 'backup'. + +# E.g if bolt and https connectors should use the same policy, the following could be declared +# dbms.ssl.policy.bolt.base_directory=certificates/default +# dbms.ssl.policy.https.base_directory=certificates/default +# However, it's strongly encouraged to not use the same key pair for multiple scopes. +# +# N.B: Note that a connector must be configured to support/require +# SSL/TLS for the policy to actually be utilized. +# +# see: dbms.connector.*.tls_level + +# SSL settings (dbms.ssl.policy..*) +# .base_directory Base directory for SSL policies paths. All relative paths within the +# SSL configuration will be resolved from the base dir. +# +# .private_key A path to the key file relative to the '.base_directory'. +# +# .private_key_password The password for the private key. +# +# .public_certificate A path to the public certificate file relative to the '.base_directory'. +# +# .trusted_dir A path to a directory containing trusted certificates. +# +# .revoked_dir Path to the directory with Certificate Revocation Lists (CRLs). +# +# .verify_hostname If true, the server will verify the hostname that the client uses to connect with. In order +# for this to work, the server public certificate must have a valid CN and/or matching +# Subject Alternative Names. +# +# .client_auth How the client should be authorized. Possible values are: 'none', 'optional', 'require'. +# +# .tls_versions A comma-separated list of allowed TLS versions. By default only TLSv1.2 is allowed. +# +# .trust_all Setting this to 'true' will ignore the trust truststore, trusting all clients and servers. +# Use of this mode is discouraged. It would offer encryption but no security. +# +# .ciphers A comma-separated list of allowed ciphers. The default ciphers are the defaults of +# the JVM platform. + +# Bolt SSL configuration +#dbms.ssl.policy.bolt.enabled=true +#dbms.ssl.policy.bolt.base_directory=certificates/bolt +#dbms.ssl.policy.bolt.private_key=private.key +#dbms.ssl.policy.bolt.public_certificate=public.crt +#dbms.ssl.policy.bolt.client_auth=NONE + +# Https SSL configuration +#dbms.ssl.policy.https.enabled=true +#dbms.ssl.policy.https.base_directory=certificates/https +#dbms.ssl.policy.https.private_key=private.key +#dbms.ssl.policy.https.public_certificate=public.crt +#dbms.ssl.policy.https.client_auth=NONE + +# Cluster SSL configuration +#dbms.ssl.policy.cluster.enabled=true +#dbms.ssl.policy.cluster.base_directory=certificates/cluster +#dbms.ssl.policy.cluster.private_key=private.key +#dbms.ssl.policy.cluster.public_certificate=public.crt + +# Backup SSL configuration +#dbms.ssl.policy.backup.enabled=true +#dbms.ssl.policy.backup.base_directory=certificates/backup +#dbms.ssl.policy.backup.private_key=private.key +#dbms.ssl.policy.backup.public_certificate=public.crt + +#***************************************************************** +# Logging configuration +#***************************************************************** + +# To enable HTTP logging, uncomment this line +#dbms.logs.http.enabled=true + +# To enable GC Logging, uncomment this line +#server.logs.gc.enabled=true + +# GC Logging Options +# see https://docs.oracle.com/en/java/javase/11/tools/java.html#GUID-BE93ABDC-999C-4CB5-A88B-1994AAAC74D5 +#server.logs.gc.options=-Xlog:gc*,safepoint,age*=trace + +# Number of GC logs to keep. +#server.logs.gc.rotation.keep_number=5 + +# Size of each GC log that is kept. +#server.logs.gc.rotation.size=20m + +#***************************************************************** +# Miscellaneous configuration +#***************************************************************** + +# Determines if Cypher will allow using file URLs when loading data using +# `LOAD CSV`. Setting this value to `false` will cause Neo4j to fail `LOAD CSV` +# clauses that load data from the file system. +#dbms.security.allow_csv_import_from_file_urls=true + + +# Value of the Access-Control-Allow-Origin header sent over any HTTP or HTTPS +# connector. This defaults to '*', which allows broadest compatibility. Note +# that any URI provided here limits HTTP/HTTPS access to that URI only. +#dbms.security.http_access_control_allow_origin=* + +# Value of the HTTP Strict-Transport-Security (HSTS) response header. This header +# tells browsers that a webpage should only be accessed using HTTPS instead of HTTP. +# It is attached to every HTTPS response. Setting is not set by default so +# 'Strict-Transport-Security' header is not sent. Value is expected to contain +# directives like 'max-age', 'includeSubDomains' and 'preload'. +#dbms.security.http_strict_transport_security= + +# Retention policy for transaction logs needed to perform recovery and backups. +db.tx_log.rotation.retention_policy=2 days 2G + +# Whether or not any database on this instance are read_only by default. +# If false, individual databases may be marked as read_only using dbms.database.read_only. +# If true, individual databases may be marked as writable using dbms.databases.writable. +#dbms.databases.default_to_read_only=false + +# Comma separated list of JAX-RS packages containing JAX-RS resources, one +# package name for each mountpoint. The listed package names will be loaded +# under the mountpoints specified. Uncomment this line to mount the +# org.neo4j.examples.server.unmanaged.HelloWorldResource.java from +# neo4j-server-examples under /examples/unmanaged, resulting in a final URL of +# http://localhost:7474/examples/unmanaged/helloworld/{nodeId} +#server.unmanaged_extension_classes=org.neo4j.examples.server.unmanaged=/examples/unmanaged + +# A comma separated list of procedures and user defined functions that are allowed +# full access to the database through unsupported/insecure internal APIs. +#dbms.security.procedures.unrestricted=my.extensions.example,my.procedures.* + +# A comma separated list of procedures to be loaded by default. +# Leaving this unconfigured will load all procedures found. +#dbms.security.procedures.allowlist=apoc.coll.*,apoc.load.*,gds.* + +#******************************************************************** +# JVM Parameters +#******************************************************************** + +# G1GC generally strikes a good balance between throughput and tail +# latency, without too much tuning. +server.jvm.additional=-XX:+UseG1GC + +# Have common exceptions keep producing stack traces, so they can be +# debugged regardless of how often logs are rotated. +server.jvm.additional=-XX:-OmitStackTraceInFastThrow + +# Make sure that `initmemory` is not only allocated, but committed to +# the process, before starting the database. This reduces memory +# fragmentation, increasing the effectiveness of transparent huge +# pages. It also reduces the possibility of seeing performance drop +# due to heap-growing GC events, where a decrease in available page +# cache leads to an increase in mean IO response time. +# Try reducing the heap memory, if this flag degrades performance. +server.jvm.additional=-XX:+AlwaysPreTouch + +# Trust that non-static final fields are really final. +# This allows more optimizations and improves overall performance. +# NOTE: Disable this if you use embedded mode, or have extensions or dependencies that may use reflection or +# serialization to change the value of final fields! +server.jvm.additional=-XX:+UnlockExperimentalVMOptions +server.jvm.additional=-XX:+TrustFinalNonStaticFields + +# Disable explicit garbage collection, which is occasionally invoked by the JDK itself. +server.jvm.additional=-XX:+DisableExplicitGC + +# Restrict size of cached JDK buffers to 1 KB +server.jvm.additional=-Djdk.nio.maxCachedBufferSize=1024 + +# More efficient buffer allocation in Netty by allowing direct no cleaner buffers. +server.jvm.additional=-Dio.netty.tryReflectionSetAccessible=true + +# Exits JVM on the first occurrence of an out-of-memory error. Its preferable to restart VM in case of out of memory errors. +# server.jvm.additional=-XX:+ExitOnOutOfMemoryError + +# Expand Diffie Hellman (DH) key size from default 1024 to 2048 for DH-RSA cipher suites used in server TLS handshakes. +# This is to protect the server from any potential passive eavesdropping. +server.jvm.additional=-Djdk.tls.ephemeralDHKeySize=2048 + +# This mitigates a DDoS vector. +server.jvm.additional=-Djdk.tls.rejectClientInitiatedRenegotiation=true + +# Enable remote debugging +#server.jvm.additional=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + +# This filter prevents deserialization of arbitrary objects via java object serialization, addressing potential vulnerabilities. +# By default this filter whitelists all neo4j classes, as well as classes from the hazelcast library and the java standard library. +# These defaults should only be modified by expert users! +# For more details (including filter syntax) see: https://openjdk.java.net/jeps/290 +#server.jvm.additional=-Djdk.serialFilter=java.**;org.neo4j.**;com.neo4j.**;com.hazelcast.**;net.sf.ehcache.Element;com.sun.proxy.*;org.openjdk.jmh.**;!* + +# Increase the default flight recorder stack sampling depth from 64 to 256, to avoid truncating frames when profiling. +server.jvm.additional=-XX:FlightRecorderOptions=stackdepth=256 + +# Allow profilers to sample between safepoints. Without this, sampling profilers may produce less accurate results. +server.jvm.additional=-XX:+UnlockDiagnosticVMOptions +server.jvm.additional=-XX:+DebugNonSafepoints + +# Open modules for neo4j to allow internal access +server.jvm.additional=--add-opens=java.base/java.nio=ALL-UNNAMED +server.jvm.additional=--add-opens=java.base/java.io=ALL-UNNAMED +server.jvm.additional=--add-opens=java.base/sun.nio.ch=ALL-UNNAMED + +# Enable access to JDK vector API +# server.jvm.additional=--add-modules=jdk.incubator.vector + +# Disable logging JMX endpoint. +server.jvm.additional=-Dlog4j2.disable.jmx=true + +# Limit JVM metaspace and code cache to allow garbage collection. Used by cypher for code generation and may grow indefinitely unless constrained. +# Useful for memory constrained environments +#server.jvm.additional=-XX:MaxMetaspaceSize=1024m +#server.jvm.additional=-XX:ReservedCodeCacheSize=512m + +# Allow big methods to be JIT compiled. +# Useful for big queries and big expressions where cypher code generation can create large methods. +#server.jvm.additional=-XX:-DontCompileHugeMethods + +#******************************************************************** +# Wrapper Windows NT/2000/XP Service Properties +#******************************************************************** +# WARNING - Do not modify any of these properties when an application +# using this configuration file has been installed as a service. +# Please uninstall the service before modifying this section. The +# service can then be reinstalled. + +# Name of the service +server.windows_service_name=neo4j + +#******************************************************************** +# Other Neo4j system properties +#******************************************************************** diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..0807ea64 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,43 @@ +# check if the neo4j-test docker container is running and stop it +if [ "$(docker ps -q -f name=neo4j-test)" ]; then + echo "Stopping the existing neo4j-test container" + docker stop neo4j-test > /dev/null +fi + +# check if the neo4j-test docker container exists and remove it +if [ "$(docker ps -aq -f name=neo4j-test)" ]; then + echo "Removing the existing neo4j-test container" + docker rm neo4j-test > /dev/null +fi + +# create a new neo4j-test docker container +echo "Creating a new neo4j-test container" +docker run -d \ + --name neo4j-test \ + -p 7474:7474 \ + -p 7687:7687 \ + -e NEO4J_AUTH=none \ + -e NEO4JLABS_PLUGINS=\[\"apoc\"\] \ + neo4j:5.20.0-ubi8 + +# Read values from config file and set them as environment variables +UBKG_SERVER=$(awk -F ' = ' '/UBKG_SERVER/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//") +UBKG_ENDPOINT_VALUESET=$(awk -F ' = ' '/UBKG_ENDPOINT_VALUESET/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//") +UBKG_CODES=$(awk -F ' = ' '/UBKG_CODES/ {print $2}' src/instance/app.cfg | tr -d '[:space:]' | sed "s/^'//;s/'$//") + +# Set the test config file and backup the original config file +mv src/instance/app.cfg src/instance/app.cfg.bak +cp test/config/app.test.cfg src/instance/app.cfg + +echo "Running tests" +UBKG_SERVER=$UBKG_SERVER \ +UBKG_ENDPOINT_VALUESET=$UBKG_ENDPOINT_VALUESET \ +UBKG_CODES=$UBKG_CODES \ +pytest -W ignore::DeprecationWarning + +# Restore the original config file +mv src/instance/app.cfg.bak src/instance/app.cfg + +echo "Stopping and removing the neo4j-test container" +docker stop neo4j-test > /dev/null +docker rm --volumes neo4j-test > /dev/null diff --git a/src/app.py b/src/app.py index 0cdc381d..8e064edb 100644 --- a/src/app.py +++ b/src/app.py @@ -5517,6 +5517,9 @@ def verify_ubkg_properties(json_data_dict): if 'intended_organ' in json_data_dict: compare_property_against_ubkg(ORGAN_TYPES, json_data_dict, 'intended_organ') + if 'intended_source_type' in json_data_dict: + compare_property_against_ubkg(SOURCE_TYPES, json_data_dict, 'intended_source_type') + def compare_property_list_against_ubkg(ubkg_dict, json_data_dict, field): good_fields = [] passes_ubkg_validation = True diff --git a/src/instance/app.cfg.example b/src/instance/app.cfg.example index da0e0d07..12a591fb 100644 --- a/src/instance/app.cfg.example +++ b/src/instance/app.cfg.example @@ -70,4 +70,4 @@ UBKG_SERVER = UBKG_ENDPOINT_VALUESET = UBKG_CODES = -MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU'] \ No newline at end of file +MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM', 'AD', 'BX', 'MU'] diff --git a/src/requirements.txt b/src/requirements.txt index 7d1ab8a0..8d21468c 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -11,7 +11,7 @@ nested-lookup==0.2.22 # The commons package requires requests>=2.22.0 and PyYAML>=5.3.1 requests==2.32.3 -PyYAML==5.4.1 +PyYAML==6.0 # Pinning specific version of neo4j neo4j==5.22.0 @@ -21,6 +21,6 @@ neo4j==5.22.0 # Default is main branch specified in docker-compose.development.yml if not set # git+https://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons hubmap-commons==2.1.18 -atlas-consortia-commons==1.0.10 +atlas-consortia-commons==1.1.0 deepdiff~=6.6.0 diff --git a/src/schema/provenance_schema.yaml b/src/schema/provenance_schema.yaml index f2cfd7d6..5348b7de 100644 --- a/src/schema/provenance_schema.yaml +++ b/src/schema/provenance_schema.yaml @@ -355,8 +355,17 @@ ENTITIES: before_entity_create_validator: validate_application_header_before_entity_create excluded_properties_from_public_response: - lab_dataset_id + - direct_ancestors: + - lab_tissue_sample_id + - source: + - lab_source_id + - origin_samples: + - lab_tissue_sample_id - sources: - lab_source_id + - metadata: + - lab_id + - slide_id - ingest_metadata: - metadata: - lab_id @@ -1290,6 +1299,11 @@ ENTITIES: type: string description: 'The organ code representing the organ type that the data contained in the upload will be registered/associated with.' required_on_create: true + intended_source_type: + type: string + description: 'The source type that the data contained in the upload will be registered/associated with.' + required_on_create: true + ############################################# EPICollection ############################################# Epicollection: diff --git a/src/schema/schema_manager.py b/src/schema/schema_manager.py index 9f7c9928..b4fb9ee6 100644 --- a/src/schema/schema_manager.py +++ b/src/schema/schema_manager.py @@ -343,7 +343,11 @@ def delete_nested_field(data, nested_path): if isinstance(value, list): for nested_field in value: if isinstance(nested_field, dict): - delete_nested_field(data[key], nested_field) + if isinstance(data[key], list): + for item in data[key]: + delete_nested_field(item, nested_field) + else: + delete_nested_field(data[key], nested_field) elif isinstance(data[key], list): for item in data[key]: diff --git a/test/config/app.cfg b/test/config/app.test.cfg similarity index 85% rename from test/config/app.cfg rename to test/config/app.test.cfg index 28af64df..461c0c60 100644 --- a/test/config/app.cfg +++ b/test/config/app.test.cfg @@ -5,14 +5,14 @@ READ_ONLY_MODE = False SCHEMA_YAML_FILE = './schema/provenance_schema.yaml' # Globus App ID and secret -APP_CLIENT_ID = 'c4018852' -APP_CLIENT_SECRET = 'supersecret' +APP_CLIENT_ID = '' +APP_CLIENT_SECRET = '' # Neo4j connection (default value used for docker localhost deployment) # Point to remote neo4j for dev/test/stage/prod deployment -NEO4J_URI = 'bolt://hubmap-neo4j-localhost:7687' +NEO4J_URI = 'bolt://localhost:7687' NEO4J_USERNAME = 'neo4j' -NEO4J_PASSWORD = '123' +NEO4J_PASSWORD = None # Set MEMCACHED_MODE to False to disable the caching for local development MEMCACHED_MODE = False @@ -46,9 +46,9 @@ GLOBUS_APP_BASE_URL = 'https://app.globus.org' # Below configurations are for DOI redirection # UUIDs of the Globus endpoints -GLOBUS_PUBLIC_ENDPOINT_UUID = '6be4ac4f-7b63-4640-9a19-3eb6d6e779d6' -GLOBUS_CONSORTIUM_ENDPOINT_UUID = '4be0eae8-ce38-41f3-bede-86d364f72201' -GLOBUS_PROTECTED_ENDPOINT_UUID = '6be4ac4f-7b63-4640-9a19-3eb6d6e779d6' +GLOBUS_PUBLIC_ENDPOINT_UUID = '' +GLOBUS_CONSORTIUM_ENDPOINT_UUID = '' +GLOBUS_PROTECTED_ENDPOINT_UUID = '' # Sub directories under the base data/globus directory where different access levels of data sits PROTECTED_DATA_SUBDIR = 'private' @@ -63,8 +63,9 @@ DOI_REDIRECT_URL = 'https://data.sennetconsortium.org/?uuid= timeout: + print("Timeout waiting for Neo4j to be ready") + raise e + print("Waiting for Neo4j to be ready...") + time.sleep(1) + + return driver + + +@pytest.fixture(scope="session") +def db_session(): + """Test fixture to create a Neo4j session + + Returns + ------- + session : neo4j.Session + Neo4j session object. Fixture takes care of closing the session. + """ + neo4j_uri = "bolt://localhost:7687" + neo4j_username = "neo4j" + neo4j_password = None + + driver = wait_for_neo4j(neo4j_uri, neo4j_username, neo4j_password) + session = driver.session() + yield session + session.close() + driver.close() + + +@pytest.fixture(scope="session") +def lab(db_session): + """Test fixture to create a lab node in the Neo4j database + + Parameters + ---------- + db_session : neo4j.Session + Neo4j session object from the `db_session` fixture + + Returns + ------- + lab : dict + Lab node properties + """ + lab = { + "entity_type": "Lab", + "last_modified_timestamp": 1661717122681, + "displayname": GROUP["displayname"], + "created_timestamp": 1661717122681, + "label": GROUP["name"], + "uuid": GROUP["uuid"], + } + + # Create a lab node if it doesn't exist + query = "MATCH (l:Lab {uuid: $uuid}) RETURN l" + result = db_session.run(query, uuid=lab["uuid"]) + if result.single() is None: + query = """ + CREATE (l:Lab { + entity_type: $entity_type, + last_modified_timestamp: $last_modified_timestamp, + displayname: $displayname, + created_timestamp: $created_timestamp, + label: $label, + uuid: $uuid + }) + """ + db_session.run(query, **lab) + + yield lab + + +def generate_entity(): + snt_first = random.randint(100, 999) + snt_second = "".join(random.choices(string.ascii_uppercase, k=4)) + snt_third = random.randint(100, 999) + sennet_id = f"SNT{snt_first}.{snt_second}.{snt_third}" + + return { + "uuid": str(uuid.uuid4()).replace("-", ""), + "sennet_id": sennet_id, + "base_id": sennet_id.replace("SNT", "").replace(".", ""), + } + + +def get_entity(uuid, db_session): + query = "MATCH (e:Entity {uuid: $uuid}) RETURN e" + result = db_session.run(query, uuid=uuid) + return result.single()["e"] + + +def create_provenance(db_session, provenance): + created_entities = {} + + previous_uuid = None + timestamp = int(time.time() * 1000) + for entity_type in provenance: + activity = generate_entity() + activity_data = { + "uuid": activity["uuid"], + "sennet_id": activity["sennet_id"], + "created_by_user_displayname": USER["name"], + "created_by_user_email": USER["email"], + "created_by_user_sub": USER["sub"], + "created_timestamp": timestamp, + "creation_action": f"Create {entity_type.title()} Activity", + "ended_at_time": timestamp, + "protocol_url": "https://dx.doi.org/tests", + "started_at_time": timestamp, + } + + entity_type = entity_type.lower() + entity = generate_entity() + data = { + "uuid": entity["uuid"], + "sennet_id": entity["sennet_id"], + "created_by_user_displayname": USER["name"], + "created_by_user_email": USER["email"], + "created_by_user_sub": USER["sub"], + "created_timestamp": timestamp, + "data_access_level": "consortium", + "group_uuid": GROUP["uuid"], + "group_name": GROUP["displayname"], + "last_modified_timestamp": timestamp, + "last_modified_user_displayname": USER["name"], + "last_modified_user_email": USER["email"], + "last_modified_user_sub": USER["sub"], + } + + if entity_type == "source": + data.update( + { + "description": "Test source description.", + "entity_type": "Source", + "lab_source_id": "test_label_source_id", + "source_type": "Human", + } + ) + elif entity_type == "organ": + data.update( + { + "description": "Test organ description.", + "entity_type": "Sample", + "lab_tissue_sample_id": "test_label_organ_sample_id", + "organ": "LI", + "sample_category": "Organ", + } + ) + elif entity_type == "block": + data.update( + { + "description": "Test block description.", + "entity_type": "Sample", + "lab_tissue_sample_id": "test_label_block_sample_id", + "sample_category": "Block", + } + ) + elif entity_type == "section": + data.update( + { + "description": "Test sample description.", + "entity_type": "Sample", + "lab_tissue_sample_id": "test_label_section_sample_id", + "sample_category": "Section", + } + ) + elif entity_type == "dataset": + data.update( + { + "contains_human_genetic_sequences": False, + "data_types": "['Visium']", + "dataset_type": "Visium (no probes)", + "entity_type": "Dataset", + "lab_dataset_id": "test_lab_dataset_id", + "method": "Test dataset method.", + "purpose": "Test dataset purpose.", + "result": "Test dataset result.", + "status": "New", + } + ) + else: + raise ValueError(f"Unknown entity type: {entity_type}") + + if previous_uuid is None: + # connect directly to lab, this is a source + db_session.run( + f"CREATE (:Entity:{data['entity_type']} {{ {', '.join(f'{k}: ${k}' for k in data)} }})", + **data, + ) + db_session.run( + "MATCH (l:Lab {uuid: $lab_uuid}), (e:Source {uuid: $source_uuid}) MERGE (l)<-[:WAS_ATTRIBUTED_TO]-(e)", + lab_uuid=GROUP["uuid"], + source_uuid=entity["uuid"], + ) + + else: + # Create and link activity + db_session.run( + f"CREATE (:Activity {{ {', '.join(f'{k}: ${k}' for k in activity_data)} }})", + **activity_data, + ) + db_session.run( + "MATCH (p:Entity {uuid: $previous_uuid}), (a:Activity {uuid: $activity_uuid}) MERGE (p)<-[:USED]-(a)", + previous_uuid=previous_uuid, + activity_uuid=activity["uuid"], + ) + # Create and link the entity + db_session.run( + f"CREATE (:Entity:{data['entity_type']} {{ {', '.join(f'{k}: ${k}' for k in data)} }})", + **data, + ) + db_session.run( + "MATCH (a:Activity {uuid: $activity_uuid}), (e:Entity {uuid: $entity_uuid}) MERGE (a)<-[:WAS_GENERATED_BY]-(e)", + activity_uuid=activity["uuid"], + entity_uuid=entity["uuid"], + ) + + previous_uuid = entity["uuid"] + + entity_data = {**data, "base_id": entity["base_id"]} + if entity_type in created_entities: + if isinstance(created_entities[entity_type], list): + created_entities[entity_type].append(entity_data) + else: + created_entities[entity_type] = [created_entities[entity_type], entity_data] + else: + created_entities[entity_type] = entity_data + + return created_entities diff --git a/test/helpers/request.py b/test/helpers/request.py new file mode 100644 index 00000000..faf29cf6 --- /dev/null +++ b/test/helpers/request.py @@ -0,0 +1,64 @@ +import pytest +import requests as requests_module + + +class RequestsMock: + def __init__(self): + self._responses = { + "get": {}, + "post": {}, + "put": {}, + "delete": {}, + } + self._call_index = { + "get": {}, + "post": {}, + "put": {}, + "delete": {}, + } + + def add_response(self, url, method, response): + normalized_url = self._normalize_url(url) + if normalized_url in self._responses[method.lower()]: + self._responses[method.lower()][normalized_url].append(response) + else: + self._responses[method.lower()][normalized_url] = [response] + + def get(self, url, *args, **kwargs): + return self._get_response(url, "get") + + def post(self, url, *args, **kwargs): + return self._get_response(url, "post") + + def put(self, url, *args, **kwargs): + return self._get_response(url, "put") + + def delete(self, url, *args, **kwargs): + return self._get_response(url, "delete") + + def _get_response(self, url, method): + normalized_url = self._normalize_url(url) + if normalized_url not in self._responses[method]: + raise ValueError(f"No response for {method.upper()} {url}") + + idx = self._call_index[method].get(normalized_url, 0) + if idx >= len(self._responses[method][normalized_url]): + raise ValueError(f"No more responses for {method.upper()} {url}") + value = self._responses[method][normalized_url][idx] + self._call_index[method][normalized_url] = idx + 1 + return value + + def _normalize_url(self, url): + return url.lower().strip("/") + + +@pytest.fixture() +def requests(monkeypatch): + mock = RequestsMock() + + monkeypatch.setattr(requests_module, "get", mock.get) + monkeypatch.setattr(requests_module, "post", mock.post) + monkeypatch.setattr(requests_module, "put", mock.put) + monkeypatch.setattr(requests_module, "delete", mock.delete) + + yield mock diff --git a/test/helpers/response.py b/test/helpers/response.py new file mode 100644 index 00000000..efd07367 --- /dev/null +++ b/test/helpers/response.py @@ -0,0 +1,9 @@ +from unittest.mock import MagicMock + + +def mock_response(status_code=200, json_data=None): + res = MagicMock() + res.status_code = status_code + if json_data: + res.json.return_value = json_data + return res diff --git a/test/test_app.py b/test/test_app.py deleted file mode 100644 index 237ff457..00000000 --- a/test/test_app.py +++ /dev/null @@ -1,409 +0,0 @@ -import json -import os -import random -import test.utils as test_utils -from unittest.mock import MagicMock, patch - -import pytest -from flask import Response - -import app as app_module - -test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - - -@pytest.fixture() -def app(): - a = app_module.app - a.config.update({'TESTING': True}) - # other setup - yield a - # clean up - - -@pytest.fixture(scope='session', autouse=True) -def ontology_mock(): - """Automatically add ontology mock functions to all tests""" - with (patch('atlas_consortia_commons.ubkg.ubkg_sdk.UbkgSDK', new=test_utils.MockOntology)): - yield - - -@pytest.fixture(scope='session', autouse=True) -def auth_helper_mock(): - auth_mock = MagicMock() - auth_mock.getUserTokenFromRequest.return_value = 'test_token' - auth_mock.getUserInfo.return_value = { - 'sub': '8cb9cda5-1930-493a-8cb9-df6742e0fb42', - 'email': 'TESTUSER@example.com', - 'hmgroupids': ['60b692ac-8f6d-485f-b965-36886ecc5a26'], - } - - # auth_helper_instance gets created (from 'import app') before fixture is called - app_module.auth_helper_instance = auth_mock - with ( - patch('hubmap_commons.hm_auth.AuthHelper.configured_instance', return_value=auth_mock), - patch('hubmap_commons.hm_auth.AuthHelper.create', return_value=auth_mock), - patch('hubmap_commons.hm_auth.AuthHelper.instance', return_value=auth_mock), - ): - yield - - -# Index - -def test_index(app): - """Test that the index page is working""" - - with app.test_client() as client: - res = client.get('/') - assert res.status_code == 200 - assert res.text == 'Hello! This is SenNet Entity API service :)' - - -# Get Entity by ID - -@pytest.mark.parametrize('entity_type', [ - ('source'), - ('sample'), - ('dataset'), -]) -def test_get_entity_by_id_success(app, entity_type): - """Test that the get entity by id endpoint returns the correct entity""" - - with open(os.path.join(test_data_dir, f'get_entity_by_id_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - - with (app.test_client() as client, - patch('app.auth_helper_instance.getUserInfo', return_value=test_data['getUserInfo']), - patch('app.auth_helper_instance.has_read_privs', return_value=test_data['has_read_privs']), - patch('app.schema_manager.get_sennet_ids', return_value=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', return_value=test_data['get_entity']), - patch('app.schema_triggers.set_dataset_sources', side_effect=test_data.get('get_associated_sources')), - patch('app.schema_neo4j_queries.get_sources_associated_entity', side_effect=test_data.get('get_associated_sources')), - patch('app.schema_manager.get_complete_entity_result', return_value=test_data['get_complete_entity_result'])): - - res = client.get(f'/entities/{entity_id}', - headers=test_data['headers']) - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -@pytest.mark.parametrize('entity_type, query_key, query_value, status_code', [ - ('source', 'property', 'data_access_level', 200), - ('source', 'property', 'status', 400), - ('sample', 'property', 'data_access_level', 200), - ('sample', 'property', 'status', 400), - ('dataset', 'property', 'data_access_level', 200), - ('dataset', 'property', 'status', 200), - ('source', 'invalid_key', 'status', 400), - ('source', 'property', 'invalid_value', 400), -]) -def test_get_entity_by_id_query(app, entity_type, query_key, query_value, status_code): - """Test that the get entity by id endpoint can handle specific query parameters""" - - with open(os.path.join(test_data_dir, f'get_entity_by_id_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - expected_response = test_data['response'] - - with (app.test_client() as client, - patch('app.auth_helper_instance.getUserInfo', return_value=test_data['getUserInfo']), - patch('app.auth_helper_instance.has_read_privs', return_value=test_data['has_read_privs']), - patch('app.schema_manager.get_sennet_ids', return_value=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', return_value=test_data['get_entity']), - patch('app.schema_manager.get_complete_entity_result', return_value=test_data['get_complete_entity_result'])): - - res = client.get(f'/entities/{entity_id}?{query_key}={query_value}', - headers=test_data['headers']) - - assert res.status_code == status_code - if status_code == 200: - assert res.text == expected_response[query_value] - - -# Get Entity by Type - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_get_entities_by_type_success(app, entity_type): - """Test that the get entity by type endpoint calls neo4j and returns the - correct entities""" - - with open(os.path.join(test_data_dir, f'get_entity_by_type_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - - with (app.test_client() as client, - patch('app.app_neo4j_queries.get_entities_by_type', return_value=test_data['get_entities_by_type']), - patch('app.schema_neo4j_queries.get_entity_creation_action_activity', return_value=test_data.get('get_entity_creation_action_activity')), - patch('app.schema_neo4j_queries.get_sources_associated_entity', return_value=test_data['get_sources_associated_entity'])): - - res = client.get(f'/{entity_type}/entities') - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -@pytest.mark.parametrize('entity_type', [ - ('invalid_type'), -]) -def test_get_entities_by_type_invalid_type(app, entity_type): - """Test that the get entity by type endpoint returns a 400 for an invalid - entity type""" - - with (app.test_client() as client): - - res = client.get(f'/{entity_type}/entities') - - assert res.status_code == 400 - - -@pytest.mark.parametrize('entity_type, query_key, query_value, status_code', [ - ('source', 'property', 'uuid', 200), - ('sample', 'property', 'uuid', 200), - ('dataset', 'property', 'uuid', 200), - ('source', 'invalid_key', 'status', 400), - ('source', 'property', 'invalid_value', 400), -]) -def test_get_entities_by_type_query(app, entity_type, query_key, query_value, status_code): - """Test that the get entities by type endpoint can handle specific query parameters""" - - with open(os.path.join(test_data_dir, f'get_entity_by_type_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - - expected_neo4j_query = test_data['get_entities_by_type'] - if status_code == 200: - expected_neo4j_query = [entity[query_value] for entity in test_data['get_entities_by_type']] - expected_response = [entity[query_value] for entity in test_data['response']] - - with (app.test_client() as client, - patch('app.app_neo4j_queries.get_entities_by_type', return_value=expected_neo4j_query)): - - res = client.get(f'/{entity_type}/entities?{query_key}={query_value}') - - assert res.status_code == status_code - if status_code == 200: - assert res.json == expected_response - - -# Create Entity - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_create_entity_success(app, entity_type): - """Test that the create entity endpoint calls neo4j and returns the correct - response""" - - with open(os.path.join(test_data_dir, f'create_entity_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - - with (app.test_client() as client, - patch('app.schema_manager.create_sennet_ids', return_value=test_data['create_sennet_ids']), - patch('app.schema_manager.get_user_info', return_value=test_data['get_user_info']), - patch('app.schema_manager.generate_triggered_data', return_value=test_data['generate_triggered_data']), - patch('app.app_neo4j_queries.create_entity', return_value=test_data['create_entity']), - patch('app.schema_manager.get_sennet_ids', return_value=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', return_value=test_data['get_entity']), - patch('app.app_neo4j_queries.get_source_organ_count', return_value=0), - patch('app.schema_neo4j_queries.get_sources_associated_entity', return_value=test_data.get('get_sources')), - patch('requests.put', return_value=Response(status=202))): - - res = client.post(f'/entities/{entity_type}', - json=test_data['request'], - headers=test_data['headers']) - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_create_entity_invalid(app, entity_type): - """Test that the create entity endpoint returns a 400 for an invalid - request schema""" - - # purposedly load the wrong entity data to use in the request body - wrong_entity_type = random.choice([i for i in ['source', 'sample', 'dataset'] if i != entity_type]) - with open(os.path.join(test_data_dir, f'create_entity_success_{wrong_entity_type}.json'), 'r') as f: - wrong_data = json.load(f) - - with open(os.path.join(test_data_dir, f'create_entity_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - - with app.test_client() as client: - - res = client.post(f'/entities/{entity_type}', - json=wrong_data['request'], - headers=test_data['headers']) - - assert res.status_code == 400 - - -# Update Entity - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_update_entity_success(app, entity_type): - """Test that the update entity endpoint returns the correct entity""" - - with open(os.path.join(test_data_dir, f'update_entity_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - - with (app.test_client() as client, - patch('app.schema_manager.get_sennet_ids', side_effect=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', side_effect=test_data['get_entity']), - - patch('app.schema_manager.get_user_info', return_value=test_data['get_user_info']), - patch('app.schema_manager.generate_triggered_data', side_effect=test_data['generate_triggered_data']), - patch('app.app_neo4j_queries.update_entity', side_effect=test_data['update_entity']), - patch('app.schema_manager.get_complete_entity_result', side_effect=test_data['get_complete_entity_result']), - patch('app.app_neo4j_queries.get_activity_was_generated_by', return_value=test_data['get_activity_was_generated_by']), - patch('app.app_neo4j_queries.get_activity', return_value=test_data['get_activity']), - patch('app.app_neo4j_queries.get_source_organ_count', return_value=0), - patch('app.schema_neo4j_queries.get_entity_creation_action_activity', return_value='lab process'), - patch('app.schema_neo4j_queries.get_sources_associated_entity', return_value=test_data.get('get_sources')), - patch('requests.put', return_value=Response(status=202))): - - res = client.put(f'/entities/{entity_id}?return_dict=true', - json=test_data['request'], - headers=test_data['headers']) - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_update_entity_invalid(app, entity_type): - """Test that the update entity endpoint returns a 400 for an invalid - request schema""" - - # purposedly load the wrong entity data to use in the request body - wrong_entity_type = random.choice([i for i in ['source', 'sample', 'dataset'] if i != entity_type]) - with open(os.path.join(test_data_dir, f'create_entity_success_{wrong_entity_type}.json'), 'r') as f: - wrong_data = json.load(f) - - with open(os.path.join(test_data_dir, f'update_entity_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - - with (app.test_client() as client, - patch('app.schema_manager.get_sennet_ids', side_effect=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', side_effect=test_data['get_entity'])): - - res = client.put(f'/entities/{entity_id}?return_dict=true', - json=wrong_data['request'], - headers=test_data['headers']) - - assert res.status_code == 400 - - -# Get Ancestors - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_get_ancestors_success(app, entity_type): - """Test that the get ancestors endpoint returns the correct entity""" - - with open(os.path.join(test_data_dir, f'get_ancestors_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - - with (app.test_client() as client, - patch('app.auth_helper_instance.getUserInfo', return_value=test_data['getUserInfo']), - patch('app.auth_helper_instance.has_read_privs', return_value=test_data['has_read_privs']), - patch('app.schema_manager.get_sennet_ids', return_value=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', return_value=test_data['get_entity']), - patch('app.app_neo4j_queries.get_ancestors', return_value=test_data['get_ancestors']), - patch('app.schema_neo4j_queries.get_sources_associated_entity', return_value=test_data['get_sources_associated_entity'])): - - res = client.get(f'/ancestors/{entity_id}', - headers=test_data['headers']) - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -# Get Descendants - -@pytest.mark.parametrize('entity_type', [ - 'source', - 'sample', - 'dataset', -]) -def test_get_descendants_success(app, entity_type): - """Test that the get descendants endpoint returns the correct entity""" - - with open(os.path.join(test_data_dir, f'get_descendants_success_{entity_type}.json'), 'r') as f: - test_data = json.load(f) - entity_id = test_data['uuid'] - - with (app.test_client() as client, - patch('app.auth_helper_instance.getUserInfo', return_value=test_data['getUserInfo']), - patch('app.auth_helper_instance.has_read_privs', return_value=test_data['has_read_privs']), - patch('app.schema_manager.get_sennet_ids', return_value=test_data['get_sennet_ids']), - patch('app.app_neo4j_queries.get_entity', return_value=test_data['get_entity']), - patch('app.app_neo4j_queries.get_descendants', return_value=test_data['get_descendants']), - patch('app.schema_neo4j_queries.get_entity_creation_action_activity', side_effect=test_data.get('get_entity_creation_action_activity')), - patch('app.schema_neo4j_queries.get_sources_associated_entity', return_value=test_data['get_sources_associated_entity'])): - - res = client.get(f'/descendants/{entity_id}', - headers=test_data['headers']) - - assert res.status_code == 200 - assert res.json == test_data['response'] - - -# Validate constraints - -@pytest.mark.parametrize('test_name', [ - 'source', - 'sample_organ', - 'sample_organ_blood', - 'sample_block', - 'sample_section', - 'sample_suspension', - 'dataset', -]) -def test_validate_constraints_new(app, test_name): - """Test that the validate constraints endpoint returns the correct constraints""" - - with open(os.path.join(test_data_dir, f'validate_constraints_{test_name}.json'), 'r') as f: - test_data = json.load(f) - - def mock_func(func_name): - data = test_data[func_name] - if data and data.get('code'): - # code being tested uses a StatusCode enum instead of an int - data['code'] = app_module.StatusCodes(data['code']) - return data - - with (app.test_client() as client, - patch('app.get_constraints_by_ancestor', return_value=mock_func('get_constraints_by_ancestor')), - patch('app.get_constraints_by_descendant', return_value=mock_func('get_constraints_by_descendant'))): - - res = client.post('/constraints' + test_data['query_string'], - headers={'Authorization': 'Bearer test_token'}, - json=test_data['request']) - - assert res.status_code == 200 - assert res.json == test_data['response'] diff --git a/test/test_create_entities.py b/test/test_create_entities.py new file mode 100644 index 00000000..daf9928f --- /dev/null +++ b/test/test_create_entities.py @@ -0,0 +1,317 @@ +from test.helpers import GROUP, USER +from test.helpers.database import create_provenance, generate_entity, get_entity +from test.helpers.response import mock_response + +import pytest + + +@pytest.fixture() +def app(auth): + import app as app_module + + app_module.app.config.update({"TESTING": True}) + app_module.auth_helper_instance = auth + app_module.schema_manager._auth_helper = auth + # other setup + yield app_module.app + # clean up + + +def test_index(app): + """Test that the index page is working""" + + with app.test_client() as client: + res = client.get("/") + assert res.status_code == 200 + assert res.text == "Hello! This is SenNet Entity API service :)" + + +# Create Entity Tests + + +@pytest.mark.usefixtures("lab") +def test_create_source(app, requests, db_session): + entities = [ + generate_entity(), # source + generate_entity(), # activity + ] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[0]])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[1]])) + requests.add_response(f"{search_api_url}/reindex/{entities[0]['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "Testing lab notes", + "group_uuid": GROUP["uuid"], + "lab_source_id": "test_lab_source_id", + "protocol_url": "dx.doi.org/10.17504/protocols.io.3byl4j398lo5/v1", + "source_type": "Human", + } + + res = client.post( + "/entities/source?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == entities[0]["uuid"] + assert res.json["sennet_id"] == entities[0]["sennet_id"] + assert res.json["description"] == data["description"] + assert res.json["lab_source_id"] == data["lab_source_id"] + assert res.json["source_type"] == data["source_type"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + # check database + db_entity = get_entity(entities[0]["uuid"], db_session) + assert db_entity["description"] == data["description"] + assert db_entity["group_uuid"] == data["group_uuid"] + assert db_entity["lab_source_id"] == data["lab_source_id"] + assert db_entity["source_type"] == data["source_type"] + + +@pytest.mark.usefixtures("lab") +def test_create_organ_sample(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source"]) + entities = [ + generate_entity(), # organ + generate_entity(), # activity + {k: test_entities["source"][k] for k in ["uuid", "sennet_id", "base_id"]}, # source + ] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response(f"{uuid_api_url}/uuid/{entities[2]['uuid']}", "get", mock_response(200, entities[2])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[0]])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[1]])) + requests.add_response(f"{search_api_url}/reindex/{entities[0]['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "sample_category": "Organ", + "organ": "LV", + "lab_tissue_sample_id": "test_lab_tissue_organ_id", + "direct_ancestor_uuid": test_entities["source"]["uuid"], # source to link to + } + + res = client.post( + "/entities/sample?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == entities[0]["uuid"] + assert res.json["sennet_id"] == entities[0]["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == data["sample_category"] + assert res.json["organ"] == data["organ"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["organ_hierarchy"] == "Liver" + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + # check database + db_entity = get_entity(entities[0]["uuid"], db_session) + assert db_entity["sample_category"] == data["sample_category"] + assert db_entity["organ"] == data["organ"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_create_block_sample(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ"]) + entities = [ + generate_entity(), # block + generate_entity(), # activity + {k: test_entities["organ"][k] for k in ["uuid", "sennet_id", "base_id"]}, # organ + ] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response(f"{uuid_api_url}/uuid/{entities[2]['uuid']}", "get", mock_response(200, entities[2])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[0]])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[1]])) + requests.add_response(f"{search_api_url}/reindex/{entities[0]['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "sample_category": "Block", + "lab_tissue_sample_id": "test_lab_tissue_block_id", + "direct_ancestor_uuid": test_entities["organ"]["uuid"], # organ to link to + } + + res = client.post( + "/entities/sample?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == entities[0]["uuid"] + assert res.json["sennet_id"] == entities[0]["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == data["sample_category"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + # check database + db_entity = get_entity(entities[0]["uuid"], db_session) + assert db_entity["sample_category"] == data["sample_category"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_create_section_sample(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block"]) + + entities = [ + generate_entity(), # section + generate_entity(), # activity + {k: test_entities["block"][k] for k in ["uuid", "sennet_id", "base_id"]}, # block + ] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response(f"{uuid_api_url}/uuid/{entities[2]['uuid']}", "get", mock_response(200, entities[2])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[0]])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[1]])) + requests.add_response(f"{search_api_url}/reindex/{entities[0]['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "sample_category": "Section", + "lab_tissue_sample_id": "test_lab_tissue_section_id", + "direct_ancestor_uuid": test_entities["block"]["uuid"], # block to link to + } + + res = client.post( + "/entities/sample?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == entities[0]["uuid"] + assert res.json["sennet_id"] == entities[0]["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == data["sample_category"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["block"]["uuid"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + # check database + db_entity = get_entity(entities[0]["uuid"], db_session) + assert db_entity["sample_category"] == data["sample_category"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_create_dataset(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section"]) + + entities = [ + generate_entity(), # dataset + generate_entity(), # activity + {k: test_entities["section"][k] for k in ["uuid", "sennet_id", "base_id"]}, # section + ] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response(f"{uuid_api_url}/uuid/{entities[2]['uuid']}", "get", mock_response(200, entities[2])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[0]])) + requests.add_response(f"{uuid_api_url}/uuid", "post", mock_response(200, [entities[1]])) + requests.add_response(f"{search_api_url}/reindex/{entities[0]['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "contains_human_genetic_sequences": False, + "dataset_type": "RNAseq", + "direct_ancestor_uuids": [test_entities["section"]["uuid"]], # section to link to + } + + res = client.post( + "/entities/dataset?return_all_properties=true", + json=data, + headers={ + "Authorization": "Bearer test_token", + "X-SenNet-Application": "portal-ui", + }, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == entities[0]["uuid"] + assert res.json["sennet_id"] == entities[0]["sennet_id"] + assert res.json["entity_type"] == "Dataset" + assert res.json["status"] == "New" + + assert res.json["contains_human_genetic_sequences"] == data["contains_human_genetic_sequences"] + assert res.json["dataset_type"] == data["dataset_type"] + assert len(res.json["direct_ancestors"]) == 1 + assert res.json["direct_ancestors"][0]["uuid"] == test_entities["section"]["uuid"] + + assert len(res.json["sources"]) == 1 + assert res.json["sources"][0]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + # check database + db_entity = get_entity(entities[0]["uuid"], db_session) + assert db_entity["contains_human_genetic_sequences"] == data["contains_human_genetic_sequences"] + assert db_entity["dataset_type"] == data["dataset_type"] diff --git a/test/test_get_entities.py b/test/test_get_entities.py new file mode 100644 index 00000000..67dbdfc5 --- /dev/null +++ b/test/test_get_entities.py @@ -0,0 +1,424 @@ +from test.helpers import GROUP, USER +from test.helpers.database import create_provenance +from test.helpers.response import mock_response + +import pytest + + +@pytest.fixture() +def app(auth): + import app as app_module + + app_module.app.config.update({"TESTING": True}) + app_module.auth_helper_instance = auth + app_module.schema_manager._auth_helper = auth + # other setup + yield app_module.app + # clean up + + +# Get Entity Tests + + +@pytest.mark.usefixtures("lab") +def test_get_source_by_uuid(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source"]) + test_source = test_entities["source"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_source['uuid']}", + "get", + mock_response(200, {k: test_source[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_source['uuid']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_source["uuid"] + assert res.json["sennet_id"] == test_source["sennet_id"] + assert res.json["description"] == test_source["description"] + assert res.json["lab_source_id"] == test_source["lab_source_id"] + assert res.json["source_type"] == test_source["source_type"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_source_by_sennet_id(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source"]) + test_source = test_entities["source"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_source['sennet_id']}", + "get", + mock_response(200, {k: test_source[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_source['sennet_id']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_source["uuid"] + assert res.json["sennet_id"] == test_source["sennet_id"] + assert res.json["description"] == test_source["description"] + assert res.json["lab_source_id"] == test_source["lab_source_id"] + assert res.json["source_type"] == test_source["source_type"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_organ_sample_by_uuid(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ"]) + test_organ = test_entities["organ"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_organ['uuid']}", + "get", + mock_response(200, {k: test_organ[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_organ['uuid']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_organ["uuid"] + assert res.json["sennet_id"] == test_organ["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_organ["sample_category"] + assert res.json["organ"] == test_organ["organ"] + assert res.json["lab_tissue_sample_id"] == test_organ["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["organ_hierarchy"] == "Large Intestine" + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_organ_sample_by_sennet_id(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ"]) + test_organ = test_entities["organ"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_organ['sennet_id']}", + "get", + mock_response(200, {k: test_organ[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_organ['sennet_id']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_organ["uuid"] + assert res.json["sennet_id"] == test_organ["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_organ["sample_category"] + assert res.json["organ"] == test_organ["organ"] + assert res.json["lab_tissue_sample_id"] == test_organ["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["organ_hierarchy"] == "Large Intestine" + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_block_sample_by_uuid(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block"]) + test_block = test_entities["block"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_block['uuid']}", + "get", + mock_response(200, {k: test_block[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_block['uuid']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_block["uuid"] + assert res.json["sennet_id"] == test_block["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_block["sample_category"] + assert res.json["lab_tissue_sample_id"] == test_block["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_block_sample_by_sennet_id(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block"]) + test_block = test_entities["block"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_block['sennet_id']}", + "get", + mock_response(200, {k: test_block[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_block['sennet_id']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_block["uuid"] + assert res.json["sennet_id"] == test_block["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_block["sample_category"] + assert res.json["lab_tissue_sample_id"] == test_block["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_section_sample_by_uuid(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section"]) + test_section = test_entities["section"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_section['uuid']}", + "get", + mock_response(200, {k: test_section[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_section['uuid']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_section["uuid"] + assert res.json["sennet_id"] == test_section["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_section["sample_category"] + assert res.json["lab_tissue_sample_id"] == test_section["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["block"]["uuid"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_section_sample_by_sennet_id(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section"]) + test_section = test_entities["section"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_section['sennet_id']}", + "get", + mock_response(200, {k: test_section[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_section['sennet_id']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_section["uuid"] + assert res.json["sennet_id"] == test_section["sennet_id"] + assert res.json["entity_type"] == "Sample" + + assert res.json["sample_category"] == test_section["sample_category"] + assert res.json["lab_tissue_sample_id"] == test_section["lab_tissue_sample_id"] + assert res.json["direct_ancestor"]["uuid"] == test_entities["block"]["uuid"] + + assert res.json["source"]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_dataset_by_uuid(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section", "dataset"]) + test_dataset = test_entities["dataset"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_dataset['uuid']}", + "get", + mock_response(200, {k: test_dataset[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_dataset['uuid']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_dataset["uuid"] + assert res.json["sennet_id"] == test_dataset["sennet_id"] + assert res.json["entity_type"] == "Dataset" + assert res.json["status"] == "New" + + assert res.json["contains_human_genetic_sequences"] == test_dataset["contains_human_genetic_sequences"] + assert res.json["dataset_type"] == test_dataset["dataset_type"] + assert len(res.json["direct_ancestors"]) == 1 + assert res.json["direct_ancestors"][0]["uuid"] == test_entities["section"]["uuid"] + + assert len(res.json["sources"]) == 1 + assert res.json["sources"][0]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" + + +@pytest.mark.usefixtures("lab") +def test_get_dataset_by_sennet_id(db_session, app, requests): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section", "dataset"]) + test_dataset = test_entities["dataset"] + + # uuid mock responses + uuid_api_url = app.config["UUID_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_dataset['sennet_id']}", + "get", + mock_response(200, {k: test_dataset[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + + with app.test_client() as client: + res = client.get( + f"/entities/{test_dataset['sennet_id']}", + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_dataset["uuid"] + assert res.json["sennet_id"] == test_dataset["sennet_id"] + assert res.json["entity_type"] == "Dataset" + assert res.json["status"] == "New" + + assert res.json["contains_human_genetic_sequences"] == test_dataset["contains_human_genetic_sequences"] + assert res.json["dataset_type"] == test_dataset["dataset_type"] + assert len(res.json["direct_ancestors"]) == 1 + assert res.json["direct_ancestors"][0]["uuid"] == test_entities["section"]["uuid"] + + assert len(res.json["sources"]) == 1 + assert res.json["sources"][0]["uuid"] == test_entities["source"]["uuid"] + assert len(res.json["origin_samples"]) == 1 + assert res.json["origin_samples"][0]["uuid"] == test_entities["organ"]["uuid"] + + assert res.json["group_uuid"] == GROUP["uuid"] + assert res.json["group_name"] == GROUP["displayname"] + assert res.json["created_by_user_displayname"] == USER["name"] + assert res.json["created_by_user_email"] == USER["email"] + assert res.json["created_by_user_sub"] == USER["sub"] + assert res.json["data_access_level"] == "consortium" diff --git a/test/test_update_entities.py b/test/test_update_entities.py new file mode 100644 index 00000000..b982bd63 --- /dev/null +++ b/test/test_update_entities.py @@ -0,0 +1,221 @@ +from test.helpers.database import create_provenance, get_entity +from test.helpers.response import mock_response + +import pytest + + +@pytest.fixture() +def app(auth): + import app as app_module + + app_module.app.config.update({"TESTING": True}) + app_module.auth_helper_instance = auth + app_module.schema_manager._auth_helper = auth + # other setup + yield app_module.app + # clean up + + +# Update Entity Tests + + +@pytest.mark.usefixtures("lab") +def test_update_source(app, requests, db_session): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source"]) + test_source = test_entities["source"] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_source['uuid']}", + "get", + mock_response(200, {k: test_source[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + requests.add_response(f"{search_api_url}/reindex/{test_source['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "New Testing lab notes", + "lab_source_id": "new_test_lab_source_id", + } + + res = client.put( + f"/entities/{test_source['uuid']}?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_source["uuid"] + assert res.json["sennet_id"] == test_source["sennet_id"] + + assert res.json["description"] == data["description"] + assert res.json["lab_source_id"] == data["lab_source_id"] + + # check database + db_entity = get_entity(test_source["uuid"], db_session) + assert db_entity["description"] == data["description"] + assert db_entity["lab_source_id"] == data["lab_source_id"] + + +@pytest.mark.usefixtures("lab") +def test_update_organ_sample(app, requests, db_session): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ"]) + test_organ = test_entities["organ"] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_organ['uuid']}", + "get", + mock_response(200, {k: test_organ[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + requests.add_response(f"{search_api_url}/reindex/{test_organ['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "New Testing lab notes", + "lab_tissue_sample_id": "new_test_lab_tissue_organ_id", + } + + res = client.put( + f"/entities/{test_organ['uuid']}?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_organ["uuid"] + assert res.json["sennet_id"] == test_organ["sennet_id"] + + assert res.json["description"] == data["description"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + # check database + db_entity = get_entity(test_organ["uuid"], db_session) + assert db_entity["description"] == data["description"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_update_block_sample(app, requests, db_session): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block"]) + test_block = test_entities["block"] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_block['uuid']}", + "get", + mock_response(200, {k: test_block[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + requests.add_response(f"{search_api_url}/reindex/{test_block['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "New Testing lab notes", + "lab_tissue_sample_id": "new_test_lab_tissue_block_id", + } + + res = client.put( + f"/entities/{test_block['uuid']}?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_block["uuid"] + assert res.json["sennet_id"] == test_block["sennet_id"] + + assert res.json["description"] == data["description"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + # check database + db_entity = get_entity(test_block["uuid"], db_session) + assert db_entity["description"] == data["description"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_update_section_sample(app, requests, db_session): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section"]) + test_section = test_entities["section"] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_section['uuid']}", + "get", + mock_response(200, {k: test_section[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + requests.add_response(f"{search_api_url}/reindex/{test_section['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "New Testing lab notes", + "lab_tissue_sample_id": "new_test_lab_tissue_section_id", + } + + res = client.put( + f"/entities/{test_section['uuid']}?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_section["uuid"] + assert res.json["sennet_id"] == test_section["sennet_id"] + + assert res.json["description"] == data["description"] + assert res.json["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + # check database + db_entity = get_entity(test_section["uuid"], db_session) + assert db_entity["description"] == data["description"] + assert db_entity["lab_tissue_sample_id"] == data["lab_tissue_sample_id"] + + +@pytest.mark.usefixtures("lab") +def test_update_dataset(app, requests, db_session): + # Create provenance in test database + test_entities = create_provenance(db_session, ["source", "organ", "block", "section", "dataset"]) + test_dataset = test_entities["dataset"] + + # uuid and search api mock responses + uuid_api_url = app.config["UUID_API_URL"] + search_api_url = app.config["SEARCH_API_URL"] + requests.add_response( + f"{uuid_api_url}/uuid/{test_dataset['uuid']}", + "get", + mock_response(200, {k: test_dataset[k] for k in ["uuid", "sennet_id", "base_id"]}), + ) + requests.add_response(f"{search_api_url}/reindex/{test_dataset['uuid']}", "put", mock_response(202)) + + with app.test_client() as client: + data = { + "description": "New Testing lab notes", + } + + res = client.put( + f"/entities/{test_dataset['uuid']}?return_all_properties=true", + json=data, + headers={"Authorization": "Bearer test_token"}, + ) + + assert res.status_code == 200 + assert res.json["uuid"] == test_dataset["uuid"] + assert res.json["sennet_id"] == test_dataset["sennet_id"] + + assert res.json["description"] == data["description"] + + # check database + db_entity = get_entity(test_dataset["uuid"], db_session) + assert db_entity["description"] == data["description"] diff --git a/test/utils.py b/test/utils.py deleted file mode 100644 index 33392fe9..00000000 --- a/test/utils.py +++ /dev/null @@ -1,174 +0,0 @@ -from dataclasses import dataclass, fields - -from atlas_consortia_commons.object import enum_val_lower - -from lib.ontology import Ontology - - -@dataclass -class SpecimenCategories: - BLOCK: str = 'Block' - ORGAN: str = 'Organ' - SECTION: str = 'Section' - SUSPENSION: str = 'Suspension' - - -@dataclass -class Entities: - DATASET: str = 'Dataset' - PUBLICATION_ENTITY: str = 'Publication Entity' - SAMPLE: str = 'Sample' - SOURCE: str = 'Source' - - -@dataclass -class SourceTypes: - MOUSE: str = 'Mouse' - HUMAN: str = 'Human' - HUMAN_ORGANOID: str = 'Human Organoid' - MOUSE_ORGANOID: str = 'Mouse Organoid' - - -@dataclass -class OrganTypes: - AD: str = 'AD' - BD: str = 'BD' - BM: str = 'BM' - BR: str = 'BR' - BS: str = 'BS' - LI: str = 'LI' - LK: str = 'LK' - LL: str = 'LL' - LN: str = 'LN' - LO: str = 'LO' - LV: str = 'LV' - MU: str = 'MU' - OT: str = 'OT' - PA: str = 'PA' - PL: str = 'PL' - RK: str = 'RK' - RL: str = 'RL' - RO: str = 'RO' - SK: str = 'SK' - - -@dataclass -class AssayTypes: - BULKRNA: str = "bulk-RNA" - CITESEQ: str = "CITE-Seq" - CODEX: str = "CODEX" - CODEXCYTOKIT: str = "codex_cytokit" - CODEXCYTOKITV1: str = "codex_cytokit_v1" - COSMX_RNA: str = "CosMX(RNA)" - DBITSEQ: str = "DBiT-seq" - FACS__FLUORESCENCEACTIVATED_CELL_SORTING: str = "FACS-Fluorescence-activatedCellSorting" - GEOMX_RNA: str = "GeoMX(RNA)" - IMAGEPYRAMID: str = "image_pyramid" - LCMS: str = "LC-MS" - LIGHTSHEET: str = "Lightsheet" - MIBI: str = "MIBI" - MIBIDEEPCELL: str = "mibi_deepcell" - MINTCHIP: str = "Mint-ChIP" - PUBLICATION: str = "publication" - PUBLICATIONANCILLARY: str = "publication_ancillary" - SALMONRNASEQ10X: str = "salmon_rnaseq_10x" - SALMONRNASEQBULK: str = "salmon_rnaseq_bulk" - SALMONSNRNASEQ10X: str = "salmon_sn_rnaseq_10x" - SASP: str = "SASP" - SCRNASEQ: str = "scRNA-seq" - SNATACSEQ: str = "snATAC-seq" - SNRNASEQ: str = "snRNA-seq" - SNRNASEQ10XGENOMICSV3: str = "snRNAseq-10xGenomics-v3" - STAINED_SLIDES: str = "StainedSlides" - VISIUM: str = "Visium" - - -@dataclass -class DatasetTypes: - HISTOLOGY: str = "Histology" - MOLECULAR_CARTOGRAPHY: str = "Molecular Cartography" - RNASEQ: str = "RNASeq" - ATACSEQ: str = "ATACSeq" - SNARESEQ2: str = "SNARE-seq2" - PHENOCYCLER: str = "PhenoCycler" - CYCIF: str = "CyCIF" - MERFISH: str = "MERFISH" - MALDI: str = "MALDI" - _2D_IMAGING_MASS_CYTOMETRY: str = "2D Imaging Mass Cytometry" - NANOSPLITS: str = "nanoSPLITS" - AUTOFLUORESCENCE: str = "Auto-fluorescence" - CONFOCAL: str = "Confocal" - THICK_SECTION_MULTIPHOTON_MXIF: str = "Thick section Multiphoton MxIF" - SECOND_HARMONIC_GENERATION_SHG: str = "Second Harmonic Generation (SHG)" - ENHANCED_STIMULATED_RAMAN_SPECTROSCOPY_SRS: str = "Enhanced Stimulated Raman Spectroscopy (SRS)" - SIMS: str = "SIMS" - CELL_DIVE: str = "Cell DIVE" - CODEX: str = "CODEX" - LIGHTSHEET: str = "Lightsheet" - MIBI: str = "MIBI" - LCMS: str = "LC-MS" - DESI: str = "DESI" - _10X_MULTIOME: str = "10x Multiome" - VISIUM: str = "Visium" - - -class MockOntology(Ontology): - @staticmethod - def entities(): - if Ontology.Ops.as_arr and MockOntology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(Entities)] - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == str: - return [e.default for e in fields(Entities)] - if MockOntology.Ops.as_data_dict: - return {e.name: e.default for e in fields(Entities)} - return Entities - - @staticmethod - def specimen_categories(): - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(SpecimenCategories)] - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == str: - return [e.default for e in fields(SpecimenCategories)] - if MockOntology.Ops.as_data_dict: - return {e.name: e.default for e in fields(SpecimenCategories)} - return SpecimenCategories - - @staticmethod - def source_types(): - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(SourceTypes)] - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == str: - return [e.default for e in fields(SourceTypes)] - if Ontology.Ops.as_data_dict: - return {e.name: e.default for e in fields(SourceTypes)} - return SourceTypes - - @staticmethod - def assay_types(): - if Ontology.Ops.as_arr and Ontology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(AssayTypes)] - if Ontology.Ops.as_arr and Ontology.Ops.cb == str: - return [e.default for e in fields(AssayTypes)] - if Ontology.Ops.as_data_dict: - return {e.name: e.default for e in fields(AssayTypes)} - return AssayTypes - - @staticmethod - def organ_types(): - if Ontology.Ops.as_arr and MockOntology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(OrganTypes)] - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == str: - return [e.default for e in fields(OrganTypes)] - if MockOntology.Ops.as_data_dict: - return {e.name: e.default for e in fields(OrganTypes)} - return OrganTypes - - @staticmethod - def dataset_types(): - if Ontology.Ops.as_arr and MockOntology.Ops.cb == enum_val_lower: - return [e.default.lower() for e in fields(DatasetTypes)] - if MockOntology.Ops.as_arr and MockOntology.Ops.cb == str: - return [e.default for e in fields(DatasetTypes)] - if MockOntology.Ops.as_data_dict: - return {e.name.removeprefix("_"): e.default for e in fields(DatasetTypes)} - return DatasetTypes