From 7f28f9c4fddae75450a09bfb69ae387c447408b3 Mon Sep 17 00:00:00 2001 From: laureen Date: Tue, 13 Feb 2024 08:01:25 +0300 Subject: [PATCH] Automated flattening update --- .../_core/compiler/linux/compile-mysql.sh | 7 ++--- .../dim_json/sp_mamba_dim_json_create.sql | 7 ++++- .../functions/fn_mamba_calculate_agegroup.sql | 2 +- ...report_array_from_automated_json_table.sql | 5 ++-- .../fn_mamba_get_datatype_for_concept.sql | 27 ++++++++++++++----- .../fn_mamba_get_obs_value_column.sql | 16 ++++++----- .../sp_mamba_json_files_insert.sql | 9 ++++--- .../sp_mamba_write_automated_json_config.sql | 24 ++++++----------- 8 files changed, 56 insertions(+), 41 deletions(-) diff --git a/api/src/main/resources/_core/compiler/linux/compile-mysql.sh b/api/src/main/resources/_core/compiler/linux/compile-mysql.sh index e989120c..37a5e5a8 100755 --- a/api/src/main/resources/_core/compiler/linux/compile-mysql.sh +++ b/api/src/main/resources/_core/compiler/linux/compile-mysql.sh @@ -70,12 +70,13 @@ function read_config_metadata() { -- \$BEGIN "$' SET @report_data = '%s'; + CALL sp_mamba_extract_report_metadata(@report_data, '\''mamba_dim_concept_metadata'\''); SET @file_count = %d; - IF @file_count = 0 THEN + IF @file_count = 0 or @file_count > 0 THEN CALL sp_mamba_dim_json(); - SET @report_data = fn_mamba_generate_report_array_from_automated_json_table(); + CALL sp_mamba_write_automated_json_config; END IF; - CALL sp_mamba_extract_report_metadata(@report_data, '\''mamba_dim_concept_metadata'\''); + '" -- \$END " diff --git a/api/src/main/resources/_core/database/mysql/dimensions/dim_json/sp_mamba_dim_json_create.sql b/api/src/main/resources/_core/database/mysql/dimensions/dim_json/sp_mamba_dim_json_create.sql index cd48f8d6..06c7e591 100644 --- a/api/src/main/resources/_core/database/mysql/dimensions/dim_json/sp_mamba_dim_json_create.sql +++ b/api/src/main/resources/_core/database/mysql/dimensions/dim_json/sp_mamba_dim_json_create.sql @@ -2,11 +2,13 @@ CREATE TABLE mamba_dim_json ( + id INT NOT NULL AUTO_INCREMENT, report_name VARCHAR(100) NOT NULL, encounter_type_id INT NOT NULL UNIQUE, Json_data JSON, + uuid CHAR(38), - PRIMARY KEY (encounter_type_id) + PRIMARY KEY (id) ) CHARSET = UTF8MB4; @@ -16,4 +18,7 @@ CREATE INDEX mamba_dim_json_encounter_type_id_index CREATE INDEX mamba_dim_json_report_name_index ON mamba_dim_json (report_name); +CREATE INDEX mamba_dim_json_uuid_index + ON mamba_dim_json (uuid); + -- $END diff --git a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_calculate_agegroup.sql b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_calculate_agegroup.sql index ec9c9cf6..f2f5c773 100644 --- a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_calculate_agegroup.sql +++ b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_calculate_agegroup.sql @@ -7,7 +7,7 @@ CREATE FUNCTION fn_mamba_calculate_agegroup(age INT) RETURNS VARCHAR(15) BEGIN DECLARE agegroup VARCHAR(15); CASE age - WHEN age < 1 THEN agegroup = '<1'; + WHEN age < 1 THEN SET agegroup = '<1'; WHEN age between 1 and 4 THEN SET agegroup = '1-4'; WHEN age between 5 and 9 THEN SET agegroup = '5-9'; WHEN age between 10 and 14 THEN SET agegroup = '10-14'; diff --git a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_generate_report_array_from_automated_json_table.sql b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_generate_report_array_from_automated_json_table.sql index d6a5807a..3ec669f6 100644 --- a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_generate_report_array_from_automated_json_table.sql +++ b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_generate_report_array_from_automated_json_table.sql @@ -7,9 +7,10 @@ CREATE FUNCTION fn_mamba_generate_report_array_from_automated_json_table() RETUR BEGIN DECLARE report_array JSON; - SELECT JSON_OBJECT('flat_report_metadata', JSON_ARRAYAGG(JSON_EXTRACT(Json_data, '$'))) + SELECT JSON_OBJECT('flat_report_metadata',JSON_ARRAYAGG(JSON_EXTRACT(Json_data, '$'))) INTO report_array - FROM mamba_dim_json; + FROM mamba_dim_json + WHERE uuid NOT IN (SELECT DISTINCT encounter_type_uuid from mamba_dim_concept_metadata); RETURN report_array; END // diff --git a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_datatype_for_concept.sql b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_datatype_for_concept.sql index 0dca9c43..a26ec99d 100644 --- a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_datatype_for_concept.sql +++ b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_datatype_for_concept.sql @@ -7,13 +7,26 @@ CREATE FUNCTION fn_mamba_get_datatype_for_concept(conceptDatatype VARCHAR(20)) R BEGIN DECLARE mysqlDatatype VARCHAR(20); - CASE conceptDatatype - WHEN conceptDatatype IN ('Coded','N/A','Text') THEN SET mysqlDatatype = 'TEXT'; - WHEN conceptDatatype = 'Boolean' THEN SET mysqlDatatype = 'VARCHAR(50)'; - WHEN conceptDatatype = 'Date' THEN SET mysqlDatatype = 'DATE'; - WHEN conceptDatatype = 'Datetime' THEN SET mysqlDatatype = 'DATETIME'; - WHEN conceptDatatype = 'Numeric' THEN SET mysqlDatatype = 'DOUBLE'; - END CASE; + IF conceptDatatype = 'Coded' + OR conceptDatatype = 'N/A' THEN + SET mysqlDatatype = 'TEXT'; + + ELSEIF conceptDatatype = 'Text' THEN + SET mysqlDatatype = 'TEXT'; + + ELSEIF conceptDatatype = 'Boolean' THEN + SET mysqlDatatype = 'VARCHAR(30)'; + + ELSEIF conceptDatatype = 'Date' THEN + SET mysqlDatatype = 'DATE'; + + ELSEIF conceptDatatype = 'Datetime' THEN + SET mysqlDatatype = 'DATETIME'; + + ELSEIF conceptDatatype = 'Numeric' THEN + SET mysqlDatatype = 'DOUBLE'; + + END IF; RETURN mysqlDatatype; END // diff --git a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_obs_value_column.sql b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_obs_value_column.sql index 76cdb35e..10c6ceb8 100644 --- a/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_obs_value_column.sql +++ b/api/src/main/resources/_core/database/mysql/functions/fn_mamba_get_obs_value_column.sql @@ -6,14 +6,16 @@ CREATE FUNCTION fn_mamba_get_obs_value_column(conceptDatatype VARCHAR(20)) RETUR DETERMINISTIC BEGIN DECLARE obsValueColumn VARCHAR(20); + IF (conceptDatatype = 'Text' OR conceptDatatype = 'Coded' OR conceptDatatype = 'N/A' OR + conceptDatatype = 'Boolean') THEN + SET obsValueColumn = 'obs_value_text'; + ELSEIF conceptDatatype = 'Date' OR conceptDatatype = 'Datetime' THEN + SET obsValueColumn = 'obs_value_datetime'; + ELSEIF conceptDatatype = 'Numeric' THEN + SET obsValueColumn = 'obs_value_numeric'; + END IF; - CASE conceptDatatype - WHEN conceptDatatype IN ('Text','Coded','N/A','Boolean') THEN SET obsValueColumn = 'obs_value_text'; - WHEN conceptDatatype IN ('Date','Datetime') THEN SET obsValueColumn = 'obs_value_datetime'; - WHEN conceptDatatype = 'Numeric' THEN SET obsValueColumn = 'obs_value_numeric'; - END CASE; - - RETURN obsValueColumn; + RETURN (obsValueColumn); END // DELIMITER ; \ No newline at end of file diff --git a/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_json_files_insert.sql b/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_json_files_insert.sql index 6d2bee79..6f20055c 100644 --- a/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_json_files_insert.sql +++ b/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_json_files_insert.sql @@ -30,7 +30,7 @@ BEGIN END IF; SET @insert_stmt = CONCAT( - 'INSERT INTO mamba_dim_json + 'INSERT INTO mamba_dim_json(report_name,encounter_type_id,Json_data,uuid) SELECT name, encounter_type_id, @@ -40,7 +40,7 @@ BEGIN ''encounter_type_uuid'',uuid, ''concepts_locale'',locale, ''table_columns'',json_obj - ) + ), encounter_type_uuid FROM ( SELECT DISTINCT et.name, @@ -54,7 +54,7 @@ BEGIN FROM ( SELECT DISTINCT et.encounter_type_id, - LOWER(LEFT(REPLACE(REPLACE(REGEXP_REPLACE(cn.name, ''[^0-9a-zÀ-ÿ ]'', ''''), '' '', ''_''),''__'', ''_''),35)) name, + LOWER(LEFT(REPLACE(REPLACE(REGEXP_REPLACE(cn.name, ''[^0-9a-z ]'', ''''), '' '', ''_''),''__'', ''_''),35)) name, c.uuid FROM mamba_source_db.obs o INNER JOIN mamba_source_db.encounter e @@ -71,7 +71,8 @@ BEGIN AND cn.locale_preferred = 1 AND et.retired = 0 ) json_obj - ) json_obj + ) json_obj, + et.uuid as encounter_type_uuid FROM mamba_source_db.encounter_type et INNER JOIN mamba_source_db.encounter e ON e.encounter_type = et.encounter_type_id diff --git a/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_write_automated_json_config.sql b/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_write_automated_json_config.sql index 7fc758ed..3adeaae9 100644 --- a/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_write_automated_json_config.sql +++ b/api/src/main/resources/_core/database/mysql/stored-procedures/sp_mamba_write_automated_json_config.sql @@ -5,33 +5,29 @@ DELIMITER // CREATE PROCEDURE sp_mamba_write_automated_json_config() BEGIN - DECLARE countRows INT; DECLARE done INT DEFAULT FALSE; DECLARE jsonData JSON; DECLARE cur CURSOR FOR SELECT json_data FROM mamba_dim_json; DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; - SELECT COUNT(*) INTO countRows FROM mamba_dim_concept_metadata; - IF countRows = 0 THEN - SET @report_data = '{"flat_report_metadata":['; OPEN cur; FETCH cur INTO jsonData; IF NOT done THEN - SET @report_data = CONCAT(@report_data, jsonData); - FETCH cur INTO jsonData; -- Fetch next record after the first one + SET @report_data = CONCAT(@report_data, jsonData); + FETCH cur INTO jsonData; -- Fetch next record after the first one END IF; - read_loop: LOOP - IF done THEN - LEAVE read_loop; - END IF; + read_loop: LOOP + IF done THEN + LEAVE read_loop; + END IF; - SET @report_data = CONCAT(@report_data, ',', jsonData); - FETCH cur INTO jsonData; + SET @report_data = CONCAT(@report_data, ',', jsonData); + FETCH cur INTO jsonData; END LOOP; CLOSE cur; @@ -39,10 +35,6 @@ BEGIN CALL sp_mamba_extract_report_metadata(@report_data, 'mamba_dim_concept_metadata'); - ELSE - SELECT 'mamba_dim_concept_metadata is not empty, no data copied.'; - END IF; - END // DELIMITER ; \ No newline at end of file