[DT-601] Adding config for source queries in AoU (#989)

* DT-601 adding source query config in underlay * DT-601 adding source query config for condition and procedure * DT-601 adding config for drug source queries * DT-601 adding source query config for observations; * DT-601 adding more source query configuration
DataBiosphere · Sep 9, 2024 · f3fdb2f · f3fdb2f
1 parent 2b2afbb
commit f3fdb2f
Show file tree

Hide file tree

Showing 38 changed files with 529 additions and 203 deletions.
diff --git a/...rces/config/criteria/aouSR2023Q3R2/criteriaselector/conditions/startDateGroupByCount.json b/...rces/config/criteria/aouSR2023Q3R2/criteriaselector/conditions/startDateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "conditionOccurrence": {
       "values": [
-        "start_date",
-        "condition"
+        "condition_start_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/.../resources/config/criteria/aouSR2023Q3R2/criteriaselector/cpt4/startDateGroupByCount.json b/.../resources/config/criteria/aouSR2023Q3R2/criteriaselector/cpt4/startDateGroupByCount.json
@@ -3,26 +3,32 @@
   "attributes": {
     "observationOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "observation_datetime",
+        "source_concept_name"
       ]
     },
     "procedureOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "procedure_datetime",
+        "source_concept_name"
       ]
     },
     "measurementOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "measurement_datetime",
+        "source_concept_name"
       ]
     },
     "ingredientOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "drug_exposure_start_datetime",
+        "source_concept_name"
+      ]
+    },
+    "deviceOccurrence": {
+      "values": [
+        "device_exposure_start_datetime",
+        "source_concept_name"
       ]
     }
   }

diff --git a/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/devices/startDateGroupByCount.json b/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/devices/startDateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "deviceOccurrence": {
       "values": [
-        "start_date",
-        "device"
+        "device_exposure_start_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/drugs/startDateGroupByCount.json b/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/drugs/startDateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "ingredientOccurrence": {
       "values": [
-        "start_date",
-        "ingredient"
+        "drug_exposure_start_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/icd10CM/startDateGroupByCount.json b/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/icd10CM/startDateGroupByCount.json
@@ -3,26 +3,26 @@
   "attributes": {
     "conditionOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "condition_start_datetime",
+        "source_concept_name"
       ]
     },
     "measurementOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "measurement_datetime",
+        "source_concept_name"
       ]
     },
     "observationOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "observation_datetime",
+        "source_concept_name"
       ]
     },
     "procedureOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "procedure_datetime",
+        "source_concept_name"
       ]
     }
   }

diff --git a/...ources/config/criteria/aouSR2023Q3R2/criteriaselector/icd10PCS/startDateGroupByCount.json b/...ources/config/criteria/aouSR2023Q3R2/criteriaselector/icd10PCS/startDateGroupByCount.json
@@ -3,14 +3,14 @@
   "attributes": {
     "ingredientOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "drug_exposure_start_datetime",
+        "source_concept_name"
       ]
     },
     "procedureOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "procedure_datetime",
+        "source_concept_name"
       ]
     }
   }

diff --git a/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/icd9CM/startDateGroupByCount.json b/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/icd9CM/startDateGroupByCount.json
@@ -3,26 +3,26 @@
   "attributes": {
     "conditionOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "condition_start_datetime",
+        "source_concept_name"
       ]
     },
     "measurementOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "measurement_datetime",
+        "source_concept_name"
       ]
     },
     "observationOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "observation_datetime",
+        "source_concept_name"
       ]
     },
     "procedureOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "procedure_datetime",
+        "source_concept_name"
       ]
     }
   }

diff --git a/...ources/config/criteria/aouSR2023Q3R2/criteriaselector/icd9Proc/startDateGroupByCount.json b/...ources/config/criteria/aouSR2023Q3R2/criteriaselector/icd9Proc/startDateGroupByCount.json
@@ -3,14 +3,14 @@
   "attributes": {
     "ingredientOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "drug_exposure_start_datetime",
+        "source_concept_name"
       ]
     },
     "procedureOccurrence": {
       "values": [
-        "start_date",
-        "source_criteria_id"
+        "procedure_datetime",
+        "source_concept_name"
       ]
     }
   }

diff --git a/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/measurement/dateGroupByCount.json b/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/measurement/dateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "measurementOccurrence": {
       "values": [
-        "date",
-        "measurement"
+        "measurement_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/observations/dateGroupByCount.json b/...sources/config/criteria/aouSR2023Q3R2/criteriaselector/observations/dateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "observationOccurrence": {
       "values": [
-        "date",
-        "observation"
+        "observation_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/...n/resources/config/criteria/aouSR2023Q3R2/criteriaselector/observations/observations.json b/...n/resources/config/criteria/aouSR2023Q3R2/criteriaselector/observations/observations.json
@@ -70,7 +70,7 @@
   ],
   "valueConfigs": [
     {
-      "attribute": "value",
+      "attribute": "value_as_concept_name",
       "title": "Categorical value"
     }
   ],

diff --git a/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/procedures/dateGroupByCount.json b/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/procedures/dateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "procedureOccurrence": {
       "values": [
-        "date",
-        "procedure"
+        "procedure_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/visits/startDateGroupByCount.json b/...esources/config/criteria/aouSR2023Q3R2/criteriaselector/visits/startDateGroupByCount.json
@@ -3,8 +3,8 @@
   "attributes": {
     "visitOccurrence": {
       "values": [
-        "start_date",
-        "visit"
+        "visit_start_datetime",
+        "standard_concept_name"
       ]
     }
   }

diff --git a/underlay/src/main/resources/config/datamapping/aouCT/entity/person/entity.json b/underlay/src/main/resources/config/datamapping/aouCT/entity/person/entity.json
@@ -5,21 +5,51 @@
     { "name": "id", "dataType": "INT64", "valueFieldName": "person_id", "isSuppressedForExport": true },
     { "name": "person_id", "dataType": "INT64" },
     { "name": "gender_concept_id", "dataType": "INT64" },
-    { "name": "gender", "dataType": "INT64", "valueFieldName": "gender_concept_id", "displayFieldName": "gender_concept_name", "isComputeDisplayHint": true },
+    { "name": "gender", "dataType": "INT64", "valueFieldName": "gender_concept_id", "displayFieldName": "gender_concept_name", "isComputeDisplayHint": true,
+      "sourceQuery": {
+        "displayFieldName": "concept_name",
+        "displayFieldTable": "${omopDataset}.concept",
+        "displayFieldTableJoinFieldName": "concept_id"
+      }
+    },
     { "name": "date_of_birth", "dataType": "TIMESTAMP", "valueFieldName": "birth_datetime", "isComputeDisplayHint": true },
     { "name": "race_concept_id", "dataType": "INT64" },
-    { "name": "race", "dataType": "INT64", "valueFieldName": "race_concept_id", "displayFieldName": "race_concept_name", "isComputeDisplayHint": true },
+    { "name": "race", "dataType": "INT64", "valueFieldName": "race_concept_id", "displayFieldName": "race_concept_name", "isComputeDisplayHint": true,
+      "sourceQuery": {
+        "displayFieldName": "concept_name",
+        "displayFieldTable": "${omopDataset}.concept",
+        "displayFieldTableJoinFieldName": "concept_id"
+      }
+    },
     { "name": "ethnicity_concept_id", "dataType": "INT64" },
-    { "name": "ethnicity", "dataType": "INT64", "valueFieldName": "ethnicity_concept_id", "displayFieldName": "ethnicity_concept_name", "isComputeDisplayHint": true },
+    { "name": "ethnicity", "dataType": "INT64", "valueFieldName": "ethnicity_concept_id", "displayFieldName": "ethnicity_concept_name", "isComputeDisplayHint": true,
+      "sourceQuery": {
+        "displayFieldName": "concept_name",
+        "displayFieldTable": "${omopDataset}.concept",
+        "displayFieldTableJoinFieldName": "concept_id"
+      }
+    },
     { "name": "sex_at_birth_concept_id", "dataType": "INT64" },
-    { "name": "sex_at_birth", "dataType": "INT64", "valueFieldName": "sex_at_birth_concept_id", "displayFieldName": "sex_at_birth_concept_name", "isComputeDisplayHint": true },
+    { "name": "sex_at_birth", "dataType": "INT64", "valueFieldName": "sex_at_birth_concept_id", "displayFieldName": "sex_at_birth_concept_name", "isComputeDisplayHint": true,
+      "sourceQuery": {
+        "displayFieldName": "concept_name",
+        "displayFieldTable": "${omopDataset}.concept",
+        "displayFieldTableJoinFieldName": "concept_id"
+      }
+    },
     { "name": "self_reported_category_concept_id", "dataType": "INT64" },
-    { "name": "self_reported_category", "dataType": "STRING" },
+    { "name": "self_reported_category", "dataType": "INT64", "valueFieldName": "self_reported_category_concept_id", "displayFieldName": "self_reported_category", "isComputeDisplayHint": true,
+      "sourceQuery": {
+        "displayFieldName": "concept_name",
+        "displayFieldTable": "${omopDataset}.concept",
+        "displayFieldTableJoinFieldName": "concept_id"
+      }
+    },
     { "name": "year_of_birth", "dataType": "INT64", "isComputeDisplayHint": true, "isSuppressedForExport": true },
     { "name": "age", "dataType": "TIMESTAMP", "valueFieldName": "birth_datetime_not_deceased",
       "runtimeSqlFunctionWrapper": "CAST(FLOOR(TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), ${fieldSql}, DAY) / 365.25) AS INT64)",
       "runtimeDataType": "INT64", "isComputeDisplayHint": true, "isSuppressedForExport": true,
-      "displayHintRangeMin": 18, "displayHintRangeMax": 120 },
+      "displayHintRangeMin": 18, "displayHintRangeMax": 125 },
     { "name": "has_fitbit_activity_summary", "dataType": "BOOLEAN", "isSuppressedForExport": true },
     { "name": "has_fitbit_heart_rate_level", "dataType": "BOOLEAN", "isSuppressedForExport": true },
     { "name": "has_fitbit_heart_rate_summary", "dataType": "BOOLEAN", "isSuppressedForExport": true },
@@ -36,5 +66,6 @@
     { "name": "is_deceased", "dataType": "BOOLEAN", "isSuppressedForExport": true }
   ],
   "idAttribute": "id",
-  "optimizeGroupByAttributes": [ "gender", "race", "age", "ethnicity" ]
+  "optimizeGroupByAttributes": [ "gender", "race", "age", "ethnicity" ],
+  "sourceQueryTableName": "${omopDataset}.person"
 }
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/conditionOccurrence/all.sql b/underlay/src/main/resources/config/datamapping/aouRT/entity/conditionOccurrence/all.sql
@@ -5,23 +5,24 @@ SELECT
   cc.concept_name AS standard_concept_name,
   cc.concept_code AS standard_concept_code,
   cc.vocabulary_id AS standard_vocabulary,
+  cc.concept_name AS condition_concept_name,
   co.condition_start_datetime,
   co.condition_end_datetime,
+  co.condition_type_concept_id,
+  ct.concept_name as condition_type_concept_name,
   co.stop_reason,
-  co.condition_source_value,
-  co.condition_source_concept_id,
-  CAST(FLOOR(TIMESTAMP_DIFF(co.condition_start_datetime, p.birth_datetime, DAY) / 365.25) AS INT64) AS age_at_occurrence,
   co.visit_occurrence_id,
-  vo.visit_concept_id,
   vc.concept_name as visit_occurrence_concept_name,
-  co.condition_type_concept_id,
-  ct.concept_name as condition_type_concept_name,
+  co.condition_source_value,
+  co.condition_source_concept_id,
   sc.concept_name AS source_concept_name,
   sc.concept_code AS source_concept_code,
   sc.vocabulary_id AS source_vocabulary,
   co.condition_status_source_value,
   co.condition_status_concept_id,
-  cs.concept_name as condition_status_concept_name
+  cs.concept_name as condition_status_concept_name,
+  CAST(FLOOR(TIMESTAMP_DIFF(co.condition_start_datetime, p.birth_datetime, DAY) / 365.25) AS INT64) AS age_at_occurrence,
+  vo.visit_concept_id,
 FROM `${omopDataset}.condition_occurrence` AS co
 JOIN `${omopDataset}.person` AS p ON p.person_id = co.person_id
 JOIN `${omopDataset}.concept` AS cc ON cc.concept_id = co.condition_concept_id

diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/conditionOccurrence/entity.json b/underlay/src/main/resources/config/datamapping/aouRT/entity/conditionOccurrence/entity.json
@@ -71,10 +71,9 @@
         "displayFieldTableJoinFieldName": "concept_id"
       }
     },
-    { "name": "source_value", "dataType": "STRING", "valueFieldName": "condition_source_value", "isSuppressedForExport": true },
-    { "name": "source_criteria_id", "dataType": "INT64", "valueFieldName": "condition_source_concept_id", "isSuppressedForExport": true },
-    { "name": "age_at_occurrence", "dataType": "INT64", "isComputeDisplayHint": true, "displayHintRangeMin": 0, "displayHintRangeMax": 100, "isSuppressedForExport": true },
-    { "name": "visit_type", "dataType": "INT64", "valueFieldName": "visit_concept_id", "displayFieldName": "visit_occurrence_concept_name", "isComputeDisplayHint": true, "isSuppressedForExport": true }
+    { "name": "age_at_occurrence", "dataType": "INT64", "isComputeDisplayHint": true, "displayHintRangeMin": 18, "displayHintRangeMax": 125, "isSuppressedForExport": true },
+    { "name": "visit_type", "dataType": "INT64", "valueFieldName": "visit_concept_id", "displayFieldName": "visit_occurrence_concept_name", "isComputeDisplayHint": true, "isSuppressedForExport": true },
+    { "name": "condition", "dataType": "INT64", "valueFieldName": "condition_concept_id", "displayFieldName": "condition_concept_name", "isSuppressedForExport": true }
   ],
   "idAttribute": "id",
   "optimizeGroupByAttributes": [ "condition_concept_id" ],