Add COPE surveys to AoU underlay (#1017)

* add entity and group for covid vaccine survey * make surveyOccurrence select distinct * add ui config for survey versions * add entity for cope survey * fix cope name in ui config
DataBiosphere · Sep 25, 2024 · 57ad00d · 57ad00d
1 parent f4fe66e
commit 57ad00d
Show file tree

Hide file tree

Showing 18 changed files with 415 additions and 11 deletions.
diff --git a/...rc/main/resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCope/selector.json b/...rc/main/resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCope/selector.json
@@ -0,0 +1,17 @@
+{
+  "name": "tanagra-surveyCope",
+  "displayName": "COVID-19 Participant Experience (COPE) Survey",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Surveys",
+    "tags": [
+      "Standard Codes"
+    ]
+  },
+  "filterBuilder": "core.EntityGroupFilterBuilder",
+  "plugin": "entityGroup",
+  "pluginConfig": null,
+  "pluginConfigFile": "surveyCope.json",
+  "modifiers": []
+}
diff --git a/.../main/resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCope/surveyCope.json b/.../main/resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCope/surveyCope.json
@@ -0,0 +1,61 @@
+{
+  "columns": [
+    {
+      "key": "name",
+      "widthString": "100%",
+      "title": "Name"
+    },
+    {
+      "key": "concept_id",
+      "widthDouble": 120,
+      "title": "Concept Id"
+    },
+    {
+      "key": "value",
+      "widthDouble": 120,
+      "title": "Answer Concept Id"
+    },
+    {
+      "key": "subtype",
+      "widthDouble": 180,
+      "title": "Type"
+    },
+    {
+      "key": "t_rollup_count",
+      "widthDouble": 150,
+      "title": "Count"
+    }
+  ],
+  "hierarchyColumns": [
+    {
+      "key": "name",
+      "widthString": "60%",
+      "title": "Name"
+    },
+    {
+      "key": "t_rollup_count",
+      "widthString": "40%",
+      "title": "Count"
+    }
+  ],
+  "classificationEntityGroups": [
+    {
+      "id": "surveyCopePerson",
+      "sortOrder": {
+        "attribute": "id",
+        "direction": "SORT_ORDER_DIRECTION_ASCENDING"
+      }
+    }
+  ],
+  "multiSelect": false,
+  "valueConfigs": [
+    {
+      "attribute": "value_numeric",
+      "title": "Numeric value"
+    },
+    {
+      "attribute": "survey_version",
+      "title": "Survey Version"
+    }
+  ]
+}
diff --git a/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCovidVaccine/selector.json b/...resources/config/criteria/aouSR2023Q3R2/criteriaselector/surveyCovidVaccine/selector.json
@@ -0,0 +1,17 @@
+{
+  "name": "tanagra-surveyCovidVaccine",
+  "displayName": "COVID-19 Vaccine Survey",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Surveys",
+    "tags": [
+      "Standard Codes"
+    ]
+  },
+  "filterBuilder": "core.EntityGroupFilterBuilder",
+  "plugin": "entityGroup",
+  "pluginConfig": null,
+  "pluginConfigFile": "surveyCovidVaccine.json",
+  "modifiers": []
+}
diff --git a/...config/criteria/aouSR2023Q3R2/criteriaselector/surveyCovidVaccine/surveyCovidVaccine.json b/...config/criteria/aouSR2023Q3R2/criteriaselector/surveyCovidVaccine/surveyCovidVaccine.json
@@ -0,0 +1,61 @@
+{
+  "columns": [
+    {
+      "key": "name",
+      "widthString": "100%",
+      "title": "Name"
+    },
+    {
+      "key": "concept_id",
+      "widthDouble": 120,
+      "title": "Concept Id"
+    },
+    {
+      "key": "value",
+      "widthDouble": 120,
+      "title": "Answer Concept Id"
+    },
+    {
+      "key": "subtype",
+      "widthDouble": 180,
+      "title": "Type"
+    },
+    {
+      "key": "t_rollup_count",
+      "widthDouble": 150,
+      "title": "Count"
+    }
+  ],
+  "hierarchyColumns": [
+    {
+      "key": "name",
+      "widthString": "60%",
+      "title": "Name"
+    },
+    {
+      "key": "t_rollup_count",
+      "widthString": "40%",
+      "title": "Count"
+    }
+  ],
+  "classificationEntityGroups": [
+    {
+      "id": "surveyCovidVaccinePerson",
+      "sortOrder": {
+        "attribute": "id",
+        "direction": "SORT_ORDER_DIRECTION_ASCENDING"
+      }
+    }
+  ],
+  "multiSelect": false,
+  "valueConfigs": [
+    {
+      "attribute": "value_numeric",
+      "title": "Numeric value"
+    },
+    {
+      "attribute": "survey_version",
+      "title": "Survey Version"
+    }
+  ]
+}
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/all.sql b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/all.sql
@@ -0,0 +1,3 @@
+SELECT id, name, code, subtype, concept_id, CAST(value AS INT64) AS value
+FROM `${omopDataset}.prep_survey`
+WHERE survey IN ('May2020Covid19Participa', 'July2020Covid19Particip', 'February2021COVID19Part')
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/childParent.sql b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/childParent.sql
@@ -0,0 +1,3 @@
+SELECT parent_id AS parent, id AS child
+FROM `${omopDataset}.prep_survey`
+WHERE parent_id != 0 AND survey IN ('May2020Covid19Participa', 'July2020Covid19Particip', 'February2021COVID19Part')
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/entity.json b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCope/entity.json
@@ -0,0 +1,27 @@
+{
+  "name": "surveyCope",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "INT64" },
+    { "name": "name", "dataType": "STRING" },
+    { "name": "code", "dataType": "STRING" },
+    { "name": "subtype", "dataType": "STRING", "isComputeDisplayHint": true },
+    { "name": "concept_id", "dataType": "INT64" },
+    { "name": "value", "dataType": "INT64" }
+  ],
+  "idAttribute": "id",
+  "textSearch": {
+    "attributes": [ "name", "code", "subtype", "concept_id", "value" ]
+  },
+  "hierarchies": [
+    {
+      "childParentIdPairsSqlFile": "childParent.sql",
+      "childIdFieldName": "child",
+      "parentIdFieldName": "parent",
+      "rootNodeIds": [ 24000 ],
+      "maxDepth": 5,
+      "keepOrphanNodes": false,
+      "cleanHierarchyNodesWithZeroCounts": true
+    }
+  ]
+}
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/all.sql b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/all.sql
@@ -0,0 +1,3 @@
+SELECT id, name, code, subtype, concept_id, CAST(value AS INT64) AS value
+FROM `${omopDataset}.prep_survey`
+WHERE survey = 'NewYearMinuteSurveyOnCO'
diff --git a/...lay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/childParent.sql b/...lay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/childParent.sql
@@ -0,0 +1,3 @@
+SELECT parent_id AS parent, id AS child
+FROM `${omopDataset}.prep_survey`
+WHERE parent_id != 0 AND survey = 'NewYearMinuteSurveyOnCO'
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/entity.json b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyCovidVaccine/entity.json
@@ -0,0 +1,27 @@
+{
+  "name": "surveyCovidVaccine",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "INT64" },
+    { "name": "name", "dataType": "STRING" },
+    { "name": "code", "dataType": "STRING" },
+    { "name": "subtype", "dataType": "STRING", "isComputeDisplayHint": true },
+    { "name": "concept_id", "dataType": "INT64" },
+    { "name": "value", "dataType": "INT64" }
+  ],
+  "idAttribute": "id",
+  "textSearch": {
+    "attributes": [ "name", "code", "subtype", "concept_id", "value" ]
+  },
+  "hierarchies": [
+    {
+      "childParentIdPairsSqlFile": "childParent.sql",
+      "childIdFieldName": "child",
+      "parentIdFieldName": "parent",
+      "rootNodeIds": [ 42000 ],
+      "maxDepth": 5,
+      "keepOrphanNodes": false,
+      "cleanHierarchyNodesWithZeroCounts": true
+    }
+  ]
+}
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyOccurrence/all.sql b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyOccurrence/all.sql
@@ -1,4 +1,4 @@
-SELECT
+SELECT distinct
   o.observation_id,
   o.person_id,
   o.observation_datetime as survey_datetime,
@@ -31,20 +31,27 @@ JOIN `${omopDataset}.prep_survey` AS ps_survey
 LEFT JOIN `${omopDataset}.concept` AS sc
     ON sc.concept_id = ps_survey.concept_id
 LEFT JOIN `${omopDataset}.prep_survey` AS qc
-    ON qc.concept_id = o.observation_source_concept_id 
+    ON qc.concept_id = o.observation_source_concept_id
     AND qc.subtype = 'QUESTION'
 LEFT JOIN `${omopDataset}.prep_survey` AS ac
-    ON ac.concept_id = o.observation_source_concept_id 
-    AND CAST(ac.value AS INT64) = o.value_source_concept_id 
+    ON ac.concept_id = o.observation_source_concept_id
+    AND CAST(ac.value AS INT64) = o.value_source_concept_id
     AND ac.subtype = 'ANSWER'
 LEFT JOIN `${omopDataset}.survey_conduct` AS sv
     ON sv.survey_conduct_id = o.questionnaire_response_id
 LEFT JOIN `${omopDataset}.concept` AS svc
     ON svc.concept_id = sv.survey_concept_id
 
-WHERE ps.survey IN ('Basics', 'Lifestyle', 'OverallHealth', 'HealthCareAccessUtiliza', 'SocialDeterminantsOfHea')
+WHERE ps.survey IN (
+    'Basics',
+    'Lifestyle',
+    'OverallHealth',
+    'HealthCareAccessUtiliza',
+    'SocialDeterminantsOfHea'
+)
 
 UNION ALL
+-- PFHH Survey
 SELECT
     ROW_NUMBER() OVER() AS observation_id,
     po.person_id,
@@ -86,3 +93,95 @@ LEFT JOIN `${omopDataset}.prep_survey` AS ac
     AND ac.subtype = 'ANSWER'
 
 WHERE ps.survey = 'PersonalAndFamilyHealth'
+
+UNION ALL
+-- Cope Surveys
+SELECT distinct
+    o.observation_id,
+    o.person_id,
+    o.observation_datetime as survey_datetime,
+    o.value_as_number,
+    ps.id AS survey_item_id,
+    ps_survey.concept_id AS survey_concept_id,
+    sc.concept_name AS survey_concept_name,
+    o.observation_source_concept_id AS question_concept_id,
+    qc.name AS question_concept_name,
+    o.value_source_concept_id AS answer_concept_id,
+    ac.name AS answer_concept_name,
+    sv.survey_concept_id as survey_version_concept_id,
+    svc.concept_name as survey_version_name
+FROM `all-of-us-ehr-dev.SR2023Q3R2.observation` AS o
+         JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ps
+              ON ps.concept_id = o.observation_source_concept_id
+                  AND (
+                     (CAST(ps.value AS INT64) = o.value_source_concept_id)
+                         OR
+                     (ps.value IS NULL AND o.value_source_concept_id IS NULL)
+                     )
+                  AND ps.subtype = 'ANSWER'
+                  AND survey IN ('February2021COVID19Part', 'July2020Covid19Particip', 'May2020Covid19Participa')
+         JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ps_survey
+              ON ps_survey.survey = ps.survey
+                  AND ps_survey.subtype = 'SURVEY'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.concept` AS sc
+                   ON sc.concept_id = ps_survey.concept_id
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS qc
+                   ON qc.concept_id = o.observation_source_concept_id
+                       AND qc.subtype = 'QUESTION'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ac
+                   ON ac.concept_id = o.observation_source_concept_id
+                       AND CAST(ac.value AS INT64) = o.value_source_concept_id
+                       AND ac.subtype = 'ANSWER'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.survey_conduct` AS sv
+                   ON sv.survey_conduct_id = o.questionnaire_response_id
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.concept` AS svc
+                   ON svc.concept_id = sv.survey_concept_id
+WHERE sv.survey_concept_id IN (
+    2100000002, 2100000003, 2100000004, 2100000005, 2100000006, 2100000007
+)
+
+UNION ALL
+-- Cope Minute Vaccine Surveys
+SELECT distinct
+    o.observation_id,
+    o.person_id,
+    o.observation_datetime as survey_datetime,
+    o.value_as_number,
+    ps.id AS survey_item_id,
+    ps_survey.concept_id AS survey_concept_id,
+    sc.concept_name AS survey_concept_name,
+    o.observation_source_concept_id AS question_concept_id,
+    qc.name AS question_concept_name,
+    o.value_source_concept_id AS answer_concept_id,
+    ac.name AS answer_concept_name,
+    sv.survey_concept_id as survey_version_concept_id,
+    svc.concept_name as survey_version_name
+FROM `all-of-us-ehr-dev.SR2023Q3R2.observation` AS o
+         JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ps
+              ON ps.concept_id = o.observation_source_concept_id
+                  AND (
+                     (CAST(ps.value AS INT64) = o.value_source_concept_id)
+                         OR
+                     (ps.value IS NULL AND o.value_source_concept_id IS NULL)
+                     )
+                  AND ps.subtype = 'ANSWER'
+                  AND survey IN ('NewYearMinuteSurveyOnCO')
+         JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ps_survey
+              ON ps_survey.survey = ps.survey
+                  AND ps_survey.subtype = 'SURVEY'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.concept` AS sc
+                   ON sc.concept_id = ps_survey.concept_id
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS qc
+                   ON qc.concept_id = o.observation_source_concept_id
+                       AND qc.subtype = 'QUESTION'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.prep_survey` AS ac
+                   ON ac.concept_id = o.observation_source_concept_id
+                       AND CAST(ac.value AS INT64) = o.value_source_concept_id
+                       AND ac.subtype = 'ANSWER'
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.survey_conduct` AS sv
+                   ON sv.survey_conduct_id = o.questionnaire_response_id
+         LEFT JOIN `all-of-us-ehr-dev.SR2023Q3R2.concept` AS svc
+                   ON svc.concept_id = sv.survey_concept_id
+WHERE sv.survey_concept_id IN (
+    905047, 905055, 765936, 1741006
+)
diff --git a/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyOccurrence/entity.json b/underlay/src/main/resources/config/datamapping/aouRT/entity/surveyOccurrence/entity.json
@@ -10,8 +10,7 @@
     { "name": "question", "dataType": "INT64", "valueFieldName": "question_concept_id", "displayFieldName": "question_concept_name" },
     { "name": "answer_concept_id", "dataType": "INT64" },
     { "name": "answer", "dataType": "INT64", "valueFieldName": "answer_concept_id", "displayFieldName": "answer_concept_name" },
-    { "name": "survey_version_concept_id", "dataType": "INT64" },
-    { "name": "survey_version_name", "dataType": "STRING" },
+    { "name": "survey_version", "dataType": "INT64", "valueFieldName": "survey_version_concept_id", "displayFieldName": "survey_version_name" },
     { "name": "value_numeric", "dataType": "DOUBLE", "valueFieldName": "value_as_number", "isSuppressedForExport": true },
     { "name": "survey_item_id", "dataType": "INT64", "isSuppressedForExport": true }
   ],