From bf09e6b7bb26bdd39e9442d984771c0b93fa5fe6 Mon Sep 17 00:00:00 2001 From: Inge Amlien Date: Mon, 27 May 2024 11:21:15 +0000 Subject: [PATCH] Fix for cross join with multiple repeated data + tests --- test/test_api.py | 117 ++++++++++++++++++++++++++++++++++++++ test/test_fixtures.py | 105 ++++++++++++++++++++++++++++++++++ webui/sql_build_query.php | 33 ++++++++--- 3 files changed, 247 insertions(+), 8 deletions(-) create mode 100644 test/test_api.py create mode 100644 test/test_fixtures.py diff --git a/test/test_api.py b/test/test_api.py new file mode 100644 index 0000000..742c017 --- /dev/null +++ b/test/test_api.py @@ -0,0 +1,117 @@ +import pytest +import requests +import json +import pandas as pd +from test_fixtures import run_dbimport, start_webui, setup_database + +def test_webui_landing(setup_database, run_dbimport, start_webui): + response = requests.get("http://127.0.0.1:3880/php/projects.php") + assert response.status_code == 200 + + expected_response = { + "status_ok": True, + "status_msg": "ok", + "type": "dbmeta", + "data": ["Proj1", "Proj2"], + } + assert response.json() == expected_response + +def test_api_list_vars(setup_database, run_dbimport, start_webui): + response = requests.get("http://127.0.0.1:3880/php/dbmeta.php?prj=all") + assert response.status_code == 200 + + r = response.json() + assert len(r["data"]["tables"]) > 1 + assert len(r["data"]["tables"]) < 100 + assert r["data"]["tables"][0]["id"] == "core" + +def test_api_doublesession_bug(setup_database, run_dbimport, start_webui): + url = "http://127.0.0.1:3880/php/query_json.php" + payload = { + "columns": [ + {"table_id": "core", "column_id": "subject_id"}, + {"table_id": "core", "column_id": "project_id"}, + {"table_id": "core", "column_id": "wave_code"}, + {"table_id": "core", "column_id": "subject_sex"}, + {"table_id": "core", "column_id": "visit_number"}, + {"table_id": "mri_aseg", "column_id": "volume_rh_hippocampus"}, + {"table_id": "mri_aparc", "column_id": "area_lh_bankssts"}, + ], + "set_op": "all", + "date": "2024-05-22T12:52:00.708Z", + "version": "undefined (Wed May 22 12:42:59 2024)", + "project": "Proj1", + } + headers = {"Content-Type": "application/json"} + + response = requests.post(url, data=json.dumps(payload), headers=headers) + assert response.status_code == 200 + + r = response.json() + column_def = r["data"]["column_def"] + rows = r["data"]["rows"] + headers = [col["id"] for col in column_def] + df = pd.DataFrame(rows, columns=headers) + + assert len(df[df["subject_id"] == 9900070]) == 3, "Expected 3 rows for subject 9900070" + + +def test_api_doublesession_bug_union(setup_database, run_dbimport, start_webui): + url = "http://127.0.0.1:3880/php/query_json.php" + payload = { + "columns": [ + {"table_id": "core", "column_id": "subject_id"}, + {"table_id": "core", "column_id": "project_id"}, + {"table_id": "core", "column_id": "wave_code"}, + {"table_id": "core", "column_id": "subject_sex"}, + {"table_id": "core", "column_id": "visit_number"}, + {"table_id": "mri_aseg", "column_id": "volume_rh_hippocampus"}, + {"table_id": "mri_aparc", "column_id": "area_lh_bankssts"}, + ], + "set_op": "union", + "date": "2024-05-22T12:52:00.708Z", + "version": "undefined (Wed May 22 12:42:59 2024)", + "project": "Proj1", + } + headers = {"Content-Type": "application/json"} + + response = requests.post(url, data=json.dumps(payload), headers=headers) + assert response.status_code == 200 + + r = response.json() + column_def = r["data"]["column_def"] + rows = r["data"]["rows"] + headers = [col["id"] for col in column_def] + df = pd.DataFrame(rows, columns=headers) + + assert len(df[df["subject_id"] == 9900070]) == 2, "Expected 2 rows for subject 9900070" + +def test_api_doublesession_bug_intersection(setup_database, run_dbimport, start_webui): + url = "http://127.0.0.1:3880/php/query_json.php" + payload = { + "columns": [ + {"table_id": "core", "column_id": "subject_id"}, + {"table_id": "core", "column_id": "project_id"}, + {"table_id": "core", "column_id": "wave_code"}, + {"table_id": "core", "column_id": "subject_sex"}, + {"table_id": "core", "column_id": "visit_number"}, + {"table_id": "mri_aseg", "column_id": "volume_rh_hippocampus"}, + {"table_id": "mri_aparc", "column_id": "area_lh_bankssts"}, + ], + "set_op": "intersection", + "date": "2024-05-22T12:52:00.708Z", + "version": "undefined (Wed May 22 12:42:59 2024)", + "project": "Proj1", + } + headers = {"Content-Type": "application/json"} + + response = requests.post(url, data=json.dumps(payload), headers=headers) + assert response.status_code == 200 + + r = response.json() + column_def = r["data"]["column_def"] + rows = r["data"]["rows"] + headers = [col["id"] for col in column_def] + df = pd.DataFrame(rows, columns=headers) + + assert len(df[df["subject_id"] == 9900070]) == 2, "Expected 2 rows for subject 9900070" \ No newline at end of file diff --git a/test/test_fixtures.py b/test/test_fixtures.py new file mode 100644 index 0000000..4c217a2 --- /dev/null +++ b/test/test_fixtures.py @@ -0,0 +1,105 @@ +import pytest +import subprocess +import time +import os +import requests +from dotenv import load_dotenv + +load_dotenv("config_default.txt") + +def run_make_directive(directive): + process = subprocess.run(["make", directive], capture_output=True, text=True) + print(f"Running make directive {directive}:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + if process.returncode != 0: + raise RuntimeError(f"Failed to run make directive {directive}: {process.stderr}") + return process + + +def db_isready(host, port, dbname): + process = subprocess.run( + ["3rdparty/postgresql/bin/pg_isready", "-h", host, "-p", port, "-d", dbname], + capture_output=True, + text=True, + ) + print(f"Checking if database is ready:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + return process.returncode == 0 + + +def start_database(): + db_host = os.getenv("DBHOST") + db_port = os.getenv("DBPORT") + db_name = os.getenv("DBNAME") + pgdata = os.getenv("PGDATA") + print(f"Starting database with host={db_host}, port={db_port}, dbname={db_name}, pgdata={pgdata}") + + if db_isready(db_host, db_port, db_name): + print("Database is already ready.") + return + + process = subprocess.run(["make", "dbstart"], capture_output=True, text=True) + print(f"Starting database:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + if process.returncode != 0: + raise RuntimeError(f"Failed to start the database: {process.stderr}") + + for _ in range(30): + result = db_isready(db_host, db_port, db_name) + if result: + print("Database started successfully.") + return + time.sleep(1) + + stop_database() + raise RuntimeError("Database did not start in time") + + +def import_data(): + process = subprocess.run(["make", "run_dbimport"], capture_output=True, text=True) + print(f"Importing data:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + if process.returncode != 0: + raise RuntimeError(f"Failed to import data: {process.stderr}") + return process + +def stop_database(): + process = subprocess.run(["make", "dbstop"], capture_output=True, text=True) + print(f"Stooping database:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + if process.returncode != 0: + raise RuntimeError(f"Failed to stop the database: {process.stderr}") + +def clean_database(): + process = subprocess.run(["make", "dberase"], capture_output=True, text=True) + print(f"Cleaning database:\nSTDOUT: {process.stdout}\nSTDERR: {process.stderr}") + if process.returncode != 0: + raise RuntimeError(f"Failed to clean the database: {process.stderr}") + +@pytest.fixture(scope="module", autouse=True) +def setup_database(): + clean_database() + start_database() + yield + clean_database() + +@pytest.fixture(scope="module", autouse=True) +def run_dbimport(setup_database): + output = run_make_directive("run_dbimport") + return output + +@pytest.fixture(scope="module", autouse=True) +def start_webui(): + webui_process = subprocess.Popen( + ["make", "run_webui"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + time.sleep(10) # Increase sleep time if necessary + + try: + response = requests.get("http://127.0.0.1:3880/php/projects.php") + if response.status_code != 200: + raise RuntimeError("Web UI did not start correctly") + except Exception as e: + webui_process.terminate() + webui_process.wait() + raise RuntimeError("Web UI did not start correctly: " + str(e)) + + yield webui_process + + webui_process.terminate() + webui_process.wait() diff --git a/webui/sql_build_query.php b/webui/sql_build_query.php index 0398256..219f3f1 100644 --- a/webui/sql_build_query.php +++ b/webui/sql_build_query.php @@ -129,12 +129,13 @@ function _get_repeated_groups($dbmeta, $sel_tabs, $sel_cols){ continue; } // if t is not an array, it cannot have repeated group - if (!is_array($t)) { - continue; - } + // IKA: I think this allways skips/continues, as class is never array? + // if (!is_array($t)) { + // continue; + // } // has repeated_goup? - echo get_class($t); - if (!array_key_exists("repeated_group", $t)) { + // echo get_class($t); + if (!property_exists($t, "repeated_group")) { continue; } // init result? @@ -163,15 +164,30 @@ function _get_repeated_groups($dbmeta, $sel_tabs, $sel_cols){ function _get_sql_where_repeated($dbmeta, $sel_tabs, $sel_cols) { $rgroups = _get_repeated_groups($dbmeta, $sel_tabs, $sel_cols); // generate where conditions; tab0.col=tabn.col + // IKA: Add WHERE condition AND (... OR ... IS NULL) + // This fixes cross join issue. Should probably be done in the JOINS, but when code + // is structured like it is, easier to do it here. + // Probably breaks something else ¯\_(ツ)_/¯ $sqls = array(); foreach ($rgroups as $rg) { + if (count(array_values($rg)) <= 1) { + continue; + } + array_push($sqls, " AND ("); for ($i=1; $i < count($rg); $i++) { array_push($sqls, - "OR " . $rg[0 ]["table_id"] . "." . $rg[0 ]["col_id"] . + $rg[0 ]["table_id"] . "." . $rg[0 ]["col_id"] . " = " . $rg[$i]["table_id"] . "." . $rg[$i]["col_id"] ); - } - } + }; + // Does this break anything, maybe union/intersection selection option? TODO: test + for ($i=0; $i < count($rg); $i++) { + array_push($sqls, + " OR " . $rg[$i]["table_id"] . "." . $rg[$i]["col_id"] . " IS NULL " + ); + }; + array_push($sqls,")"); + }; return join("\n", $sqls); } @@ -213,6 +229,7 @@ function _get_sql_select_repeated($dbmeta, $sel_tabs, $sel_cols) { {$sql_where_repeated} ORDER BY core.subject_id "; +// error_log($sql); return $sql; }