From 4873cbf127fc7b1fa3a2e201a3113cdeb9e61c94 Mon Sep 17 00:00:00 2001 From: minminchan Date: Mon, 20 Mar 2023 12:00:13 +0800 Subject: [PATCH 1/2] initial code for process mse --- src/process_mse/process_mse.py | 72 ++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/process_mse/process_mse.py diff --git a/src/process_mse/process_mse.py b/src/process_mse/process_mse.py new file mode 100644 index 0000000..2edf167 --- /dev/null +++ b/src/process_mse/process_mse.py @@ -0,0 +1,72 @@ +import neuroblu_postgres as nb +import jsonref +import warnings + + +def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname=None): + """ + process mse data of patients + This function returns the mse data of specified patients. + Parameters + ---------- + cohort (pd.DataFrame): + Minimum columns: person_id, index_date + The version of the schema as defined in OMOP CDM + version : CDM version + window : window period to filter MSE data + time window specified as a list [left cut-off , right cut-off] in days from the index date + columns : `str, array-like of str`, *optional* + SAVs of interest specified as str or array-like collection + of str. + If None, the function will return data for all SAV columns. + mapping : `str, array-like of str`, *optional* + SAV mappings of interest specified as a list of string/s. + mapping has to be specified with two degrees of freedom separated with the semicolon delimiter + in maincategory:subcategory format (i.e. Suicidal:suicidal_ideation) + Please see list of SAV main and subcategories in Notion Page: + https://www.notion.so/holmusk/SAV-3f7fe1348c044f599c3af27c4330582a + If None, the function will return data for all SAV columns. + dbname: `string`, *optional* + The name of the database as defined in ``db.json`` file. + If None, the defaultDB is used. + + Returns + A dataframe of SAV labels with measurement dates within the specified time window + + Example: + time_window = [-14,365] # in days + sav_mappings = ['Suicidal:"normal, no issues, not present"', 'Suicidal:"neutral, unable to categorize"', 'Suicidal:suicidal ideation'] + sav_columns = ["sav_221_suicidal_0","sav_222_suicidal_1","sav_223_suicidal_2"] + + df = process_mse(cohort, version, window, mapping = sav_mappings) + df = process_mse(cohort, version, window, columns = sav_columns) + + """ + + if mapping is not None: + if len(mapping) == 0: + raise ValueError("mapping contain no values") + if columns is not None: + raise ValueError("two types of mapping files given, please use only one") + mapping_dictionary = jsonref.load(open('/Users/chanminmin/Desktop/function_project/mapping_dictionary.json')) + for mapping_name in mapping: + if mapping_name not in list(mapping_dictionary.keys()): + raise ValueError("mapping contains invalid SAV category:subcategory names") + columns = list(map(mapping_dictionary.get, mapping) ) + + df = nb.get_SAV(cohort['person_id'].tolist(), version, columns=columns, dbname=dbname) + if df.empty: + if len(df) == 0: + raise ValueError("no person ids with sav labels") + else: + df = cohort.merge(df,on='person_id') + if df['person_id'].nunique() != cohort['person_id'].nunique(): + warnings.warn("some person_ids has no sav labels") + + if window is None: + raise TypeError("window period has to be specified") + else: + df = df[(df['measurement_date'] - df['index_date']).dt.days >= window[0]] + df = df[(df['measurement_date'] - df['index_date']).dt.days <= window[1]] + + return df From 0f2337969e433b14ca064adecac7bcd22ecbf783 Mon Sep 17 00:00:00 2001 From: minminchan Date: Mon, 20 Mar 2023 14:13:27 +0800 Subject: [PATCH 2/2] add config for sav_mapping_dictionary --- config/sav_mapping_dictionary.json | 243 +++++++++++++++++++++++++++++ src/process_mse/process_mse.py | 19 +-- 2 files changed, 253 insertions(+), 9 deletions(-) create mode 100644 config/sav_mapping_dictionary.json diff --git a/config/sav_mapping_dictionary.json b/config/sav_mapping_dictionary.json new file mode 100644 index 0000000..4e08cf4 --- /dev/null +++ b/config/sav_mapping_dictionary.json @@ -0,0 +1,243 @@ +{ + "Abnormal or psychotic thoughts:no issues": "sav_0_aopt_0", + "Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (grandure)": "sav_1_aopt_1", + "Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (persecution)": "sav_2_aopt_2", + "Abnormal or psychotic thoughts:experiencing delusions/abnromal thoughts (religious)": "sav_3_aopt_3", + "Abnormal or psychotic thoughts:experiencing hallucinations (auditory)": "sav_4_aopt_4", + "Abnormal or psychotic thoughts:experiencing hallucinations (visual)": "sav_5_aopt_5", + "Abnormal or psychotic thoughts:experiencing hallucinations (tactile)": "sav_6_aopt_6", + "Abnormal or psychotic thoughts:History of delusions/abnormal thoughts": "sav_7_aopt_7", + "Abnormal or psychotic thoughts:History of hallucinations": "sav_8_aopt_8", + "Abnormal or psychotic thoughts:neutral/unable to categorize": "sav_9_aopt_9", + "Abnormal or psychotic thoughts:experiencing delusions (not specified)": "sav_10_aopt_10", + "Abnormal or psychotic thoughts:experiencing hallucinations (not specified)": "sav_11_aopt_11", + "Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (obsessions)": "sav_12_aopt_12", + "Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (paranoia)": "sav_13_aopt_13", + "Abnormal or psychotic thoughts:experiencing hallucinations (olfactory)": "sav_14_aopt_14", + "Abnormal or psychotic thoughts:responding to internal stimuli": "sav_15_aopt_15", + "Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (sexual)": "sav_16_aopt_16", + "Affect:no issues/appropriate affect": "sav_17_affect_0", + "Affect:neutral/unable to categorize": "sav_18_affect_1", + "Affect:aggressive": "sav_19_affect_2", + "Affect:reactive": "sav_20_affect_3", + "Affect:irritable/angry": "sav_21_affect_4", + "Affect:improved from last visit": "sav_22_affect_5", + "Affect:issues with cognition": "sav_23_affect_6", + "Affect:loss of energy/tiredness": "sav_24_affect_7", + "Affect:blunted/restricted": "sav_25_affect_8", + "Affect:anxious/fearful": "sav_26_affect_9", + "Affect:tearful/depressed/sad": "sav_27_affect_10", + "Affect:bright/positive": "sav_28_affect_11", + "Affect:labile": "sav_29_affect_12", + "Affect:intense": "sav_30_affect_13", + "Affect:expansive/overly animated": "sav_31_affect_14", + "Affect:generally odd or out of the oridnary/inappropriate": "sav_32_affect_15", + "Affect:flooded": "sav_33_affect_16", + "Appearance:Issues with Affect": "sav_34_appearance_0", + "Appearance:Sleep/Bed": "sav_35_appearance_1", + "Appearance:Issues with Eye Contact": "sav_36_appearance_2", + "Appearance:Not age appropriate": "sav_37_appearance_3", + "Appearance:Issues with general Health": "sav_38_appearance_4", + "Appearance:Issues with Behavior": "sav_39_appearance_5", + "Appearance:Positive Affect": "sav_40_appearance_6", + "Appearance:Postivie Grooming/Hygiene": "sav_41_appearance_7", + "Appearance:Issues with Grooming/Hygiene": "sav_42_appearance_8", + "Appearance:Positive Health": "sav_43_appearance_9", + "Appearance:Issues with Weight": "sav_44_appearance_10", + "Appearance:Normal/no issues": "sav_45_appearance_11", + "Appearance:Hospital attire": "sav_46_appearance_12", + "Appearance:Positive dress": "sav_47_appearance_13", + "Appearance:Issues with dress": "sav_48_appearance_14", + "Appearance:Neutral": "sav_49_appearance_15", + "Appearance:Positive eye contact": "sav_50_appearance_16", + "Appearance:Appropriate Age": "sav_51_appearance_17", + "Appearance:Positive Behavior": "sav_52_appearance_18", + "Association:intact/no issues": "sav_53_association_0", + "Association:neutral/unable to categorize": "sav_54_association_1", + "Association:circumstantial": "sav_55_association_2", + "Association:loose": "sav_56_association_3", + "Association:tangential": "sav_57_association_4", + "Association:age appropriate": "sav_58_association_5", + "Association:improved": "sav_59_association_6", + "Association:declined": "sav_60_association_7", + "Association:disorganized": "sav_61_association_8", + "Association:general issues": "sav_62_association_9", + "Attention/Concentration:intact/no issues": "sav_63_attn_concentration_0", + "Attention/Concentration:neutral/unable to categorize": "sav_64_attn_concentration_1", + "Attention/Concentration:general issues": "sav_65_attn_concentration_2", + "Attention/Concentration:improved": "sav_66_attn_concentration_3", + "Attention/Concentration:declined": "sav_67_attn_concentration_4", + "Attention/Concentration:age appropriate": "sav_68_attn_concentration_5", + "Attention/Concentration:issues due to MH": "sav_69_attn_concentration_6", + "Attention/Concentration:issues due to DD": "sav_70_attn_concentration_7", + "Attention/Concentration:varrying": "sav_71_attn_concentration_8", + "Attitude:positive/appropriate": "sav_72_attitude_0", + "Attitude:neutral/unable to categorize": "sav_73_attitude_1", + "Attitude:age appropriate": "sav_74_attitude_2", + "Attitude:not age appropriate": "sav_75_attitude_3", + "Attitude:negative/inappropriate": "sav_76_attitude_4", + "Attitude:improved": "sav_77_attitude_5", + "Attitude:declined": "sav_78_attitude_6", + "Cognition:normal/no issues": "sav_79_cognition_0", + "Cognition:neutral/unable to categorize": "sav_80_cognition_1", + "Cognition:issues with concentration": "sav_81_cognition_2", + "Cognition:positive concentration": "sav_82_cognition_3", + "Cognition:issues with fund of knowledge": "sav_83_cognition_4", + "Cognition:positive fund of knowledge": "sav_84_cognition_5", + "Cognition:issues with attention": "sav_85_cognition_6", + "Cognition:positive attention": "sav_86_cognition_7", + "Cognition:general issues": "sav_87_cognition_8", + "Executive Functioning:in tact/noissues": "sav_88_exec_functioning_0", + "Executive Functioning:neutral/unable to categorize": "sav_89_exec_functioning_1", + "Executive Functioning:some impairment": "sav_90_exec_functioning_2", + "Executive Functioning:much impairment": "sav_91_exec_functioning_3", + "Executive Functioning:improved since last session": "sav_92_exec_functioning_4", + "Executive Functioning:declined since last session": "sav_93_exec_functioning_5", + "Executive Functioning:age appropriate": "sav_94_exec_functioning_6", + "Executive Functioning:below age level": "sav_95_exec_functioning_7", + "Executive Functioning:impaired (due to DD)": "sav_96_exec_functioning_8", + "Executive Functioning:impaired (due to TBI)": "sav_97_exec_functioning_9", + "Executive Functioning:impaired (due to other MH issue)": "sav_98_exec_functioning_10", + "Fund of knowledge:intact/no issues": "sav_99_fund_of_knowledge_0", + "Fund of knowledge:neutral/unable to categorize": "sav_100_fund_of_knowledge_1", + "Fund of knowledge:positive vocabulary": "sav_101_fund_of_knowledge_2", + "Fund of knowledge:issues with vocabulary": "sav_102_fund_of_knowledge_3", + "Fund of knowledge:positive history": "sav_103_fund_of_knowledge_4", + "Fund of knowledge:issues with history": "sav_104_fund_of_knowledge_5", + "Fund of knowledge:average": "sav_105_fund_of_knowledge_6", + "Fund of knowledge:above average": "sav_106_fund_of_knowledge_7", + "Fund of knowledge:below average": "sav_107_fund_of_knowledge_8", + "Fund of knowledge:positive current events": "sav_108_fund_of_knowledge_9", + "Fund of knowledge:issues with current events": "sav_109_fund_of_knowledge_10", + "Fund of knowledge:generally limited/issues": "sav_110_fund_of_knowledge_11", + "Fund of knowledge:declined": "sav_111_fund_of_knowledge_12", + "Fund of knowledge:improved": "sav_112_fund_of_knowledge_13", + "Gait and station:normal/no issues": "sav_113_gait_and_station_0", + "Gait and station:neutral/unable to categorize": "sav_114_gait_and_station_1", + "Gait and station:non-ambulatory": "sav_115_gait_and_station_2", + "Gait and station:issues with gait/station": "sav_116_gait_and_station_3", + "Gait and station:assistive device needed": "sav_117_gait_and_station_4", + "Homicidal:not present": "sav_118_homicidal_0", + "Homicidal:neutral": "sav_119_homicidal_1", + "Homicidal:present": "sav_120_homicidal_2", + "Impulse control:good/no issues": "sav_121_impulse_control_0", + "Impulse control:neutral/unable to categorize": "sav_122_impulse_control_1", + "Impulse control:limited/some issues": "sav_123_impulse_control_2", + "Impulse control:poor/serious issues": "sav_124_impulse_control_3", + "Impulse control:age appropriate": "sav_125_impulse_control_4", + "Impulse control:not age appropriate": "sav_126_impulse_control_5", + "Insight:no issues with insight": "sav_127_insight_0", + "Insight:issues with insight": "sav_128_insight_1", + "Insight:neutral/unable to categorize": "sav_129_insight_2", + "Intelligence:normal/no issues with intelligence": "sav_130_intelligence_0", + "Intelligence:issues with intelligence": "sav_131_intelligence_1", + "Intelligence:neutral/unable to categorize": "sav_132_intelligence_2", + "Judgment:no issues with judgement": "sav_133_judgement_0", + "Judgment:issues with judgement": "sav_134_judgement_1", + "Judgment:neutral/unable to categorize": "sav_135_judgement_2", + "Language:intact": "sav_136_language_0", + "Language:neutral/unable to categorize": "sav_137_language_1", + "Language:repetition intact": "sav_138_language_2", + "Language:issues with repetition": "sav_139_language_3", + "Language:object naming intact": "sav_140_language_4", + "Language:issues with object naming": "sav_141_language_5", + "Language:impaired": "sav_142_language_6", + "Language:non-verbal/mute": "sav_143_language_7", + "Language:minimally verbal": "sav_144_language_8", + "Language:issues related to DD": "sav_145_language_9", + "Level of consciousness:normal; no issues notes": "sav_146_conciousness_level_0", + "Level of consciousness:neutral; unable to categorize": "sav_147_conciousness_level_1", + "Level of consciousness:sleepy/drowsy": "sav_148_conciousness_level_2", + "Level of consciousness:clouded": "sav_149_conciousness_level_3", + "Level of consciousness:issues verbally": "sav_150_conciousness_level_4", + "Level of consciousness:misc. issues": "sav_151_conciousness_level_5", + "Level of consciousness:impaired due to DD/MI": "sav_152_conciousness_level_6", + "Memory:no issues/memory unimpaired": "sav_153_memory_0", + "Memory:issues/memory impaired": "sav_154_memory_1", + "Memory:neutral/unable to categorize": "sav_155_memory_2", + "Mood:Normal; no issues": "sav_156_mood_0", + "Mood:neutral; unable to categorize": "sav_157_mood_1", + "Mood:\"anxious, tense\"": "sav_158_mood_2", + "Mood:\"depressed, sad, despondent\"": "sav_159_mood_3", + "Mood:\"irritable, angry\"": "sav_160_mood_4", + "Mood:improved": "sav_161_mood_5", + "Mood:declined": "sav_162_mood_6", + "Mood:\"elevated, manic\"": "sav_163_mood_7", + "Mood:labile": "sav_164_mood_8", + "Mood:misc. issues": "sav_165_mood_9", + "Mood:suicidal": "sav_166_mood_10", + "Orientation:no orientation issues": "sav_167_orientation_0", + "Orientation:issues with orientation to person": "sav_168_orientation_1", + "Orientation:issues with orientation to place": "sav_169_orientation_2", + "Orientation:issues with orientation to time/date": "sav_170_orientation_3", + "Orientation:issues with orientation to situation": "sav_171_orientation_4", + "Orientation:neutral/unable to categorize": "sav_172_orientation_5", + "Orientation:general orientation issues": "sav_173_orientation_6", + "Psychomotor:normal; no issues": "sav_174_psychomotor_0", + "Psychomotor:neutral; unable to categorize": "sav_175_psychomotor_1", + "Psychomotor:retarded; slowed": "sav_176_psychomotor_2", + "Psychomotor:agitated; tense; restless": "sav_177_psychomotor_3", + "Psychomotor:improved": "sav_178_psychomotor_4", + "Psychomotor:declined": "sav_179_psychomotor_5", + "Psychomotor:constricted": "sav_180_psychomotor_6", + "Psychomotor:tremors": "sav_181_psychomotor_7", + "Psychomotor:tics": "sav_182_psychomotor_8", + "Psychomotor:involuntary movements": "sav_183_psychomotor_9", + "Psychomotor:misc. issues": "sav_184_psychomotor_10", + "Psychomotor:catatonic": "sav_185_psychomotor_11", + "Reasoning:normal; no issues": "sav_186_reasoning_0", + "Reasoning:neutral; unable to categorize": "sav_187_reasoning_1", + "Reasoning:impaired; poor": "sav_188_reasoning_2", + "Reasoning:impaired due to DD/cognitive issues": "sav_189_reasoning_3", + "Reasoning:impaired due to MI": "sav_190_reasoning_4", + "Reasoning:concrete": "sav_191_reasoning_5", + "Reasoning:abstract": "sav_192_reasoning_6", + "Reasoning:improved": "sav_193_reasoning_7", + "Reasoning:misc. issues": "sav_194_reasoning_8", + "Sensorium:normal; no issues": "sav_195_sensorium_0", + "Sensorium:neutral; unable to categorize": "sav_196_sensorium_1", + "Sensorium:clouded; sedate": "sav_197_sensorium_2", + "Sensorium:impaired": "sav_198_sensorium_3", + "Sensorium:fluctuating": "sav_199_sensorium_4", + "Sensorium:impaired due to medications/drugs/alcohol": "sav_200_sensorium_5", + "Sensorium:sleepy; lethargic": "sav_201_sensorium_6", + "Sleep:\"normal, no issues\"": "sav_202_sleep_0", + "Sleep:\"neutral, unable to categorize\"": "sav_203_sleep_1", + "Sleep:maintenance issues (can't stay asleep)": "sav_204_sleep_2", + "Sleep:onset issues (unable to fall asleep)": "sav_205_sleep_3", + "Sleep:nightmares": "sav_206_sleep_4", + "Sleep:insomnia": "sav_207_sleep_5", + "Sleep:hypersomnia": "sav_208_sleep_6", + "Sleep:misc. issues related to sleep": "sav_209_sleep_7", + "Speech:\"normal, no issues\"": "sav_210_speech_0", + "Speech:\"neutral, unable to categorize\"": "sav_211_speech_1", + "Speech:issues with rate": "sav_212_speech_2", + "Speech:issues with tone": "sav_213_speech_3", + "Speech:issues with volume": "sav_214_speech_4", + "Speech:stutter or lisp": "sav_215_speech_5", + "Speech:misc. issues": "sav_216_speech_6", + "Speech:impoverished": "sav_217_speech_7", + "Speech:improved": "sav_218_speech_8", + "Speech:unintelligible": "sav_219_speech_9", + "Speech:\"mute, non-verbal\"": "sav_220_speech_10", + "Suicidal:\"normal, no issues, not present\"": "sav_221_suicidal_0", + "Suicidal:\"neutral, unable to categorize\"": "sav_222_suicidal_1", + "Suicidal:suicidal ideation": "sav_223_suicidal_2", + "Suicidal:suicidal with intent": "sav_224_suicidal_3", + "Suicidal:suicidal with plan": "sav_225_suicidal_4", + "Suicidal:suicide attempt": "sav_226_suicidal_5", + "Suicidal:history of ideation": "sav_227_suicidal_6", + "Suicidal:history of attempt": "sav_228_suicidal_7", + "Suicidal:self-injurious": "sav_229_suicidal_8", + "Suicidal:history of self injury": "sav_230_suicidal_9", + "Suicidal:suicidal ideation with means": "sav_231_suicidal_10", + "Violent Thoughts:\"normal, no issues\"": "sav_232_violent_thoughts_0", + "Violent Thoughts:\"neutral, unable to categorize\"": "sav_233_violent_thoughts_1", + "Violent Thoughts:present": "sav_234_violent_thoughts_2", + "Violent Thoughts:\"present, with intent\"": "sav_235_violent_thoughts_3", + "Violent Thoughts:present with plan": "sav_236_violent_thoughts_4", + "Violent Thoughts:present with means": "sav_237_violent_thoughts_5", + "Violent Thoughts:history of violent thoughts": "sav_238_violent_thoughts_6", + "Violent Thoughts:history of violent behaviors": "sav_239_violent_thoughts_7", + "Violent Thoughts:violent/aggressive behavior": "sav_240_violent_thoughts_8" +} \ No newline at end of file diff --git a/src/process_mse/process_mse.py b/src/process_mse/process_mse.py index 2edf167..ed84d67 100644 --- a/src/process_mse/process_mse.py +++ b/src/process_mse/process_mse.py @@ -3,7 +3,7 @@ import warnings -def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname=None): +def process_mse(cohort, version, window=None, columns=None, mapping=None, reference_col = 'index_date', dbname=None): """ process mse data of patients This function returns the mse data of specified patients. @@ -15,6 +15,7 @@ def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname version : CDM version window : window period to filter MSE data time window specified as a list [left cut-off , right cut-off] in days from the index date + if None, the function will return data for all SAV measurements columns : `str, array-like of str`, *optional* SAVs of interest specified as str or array-like collection of str. @@ -38,8 +39,8 @@ def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname sav_mappings = ['Suicidal:"normal, no issues, not present"', 'Suicidal:"neutral, unable to categorize"', 'Suicidal:suicidal ideation'] sav_columns = ["sav_221_suicidal_0","sav_222_suicidal_1","sav_223_suicidal_2"] - df = process_mse(cohort, version, window, mapping = sav_mappings) - df = process_mse(cohort, version, window, columns = sav_columns) + df = process_mse(cohort[["person_id","index_date"]], version, window=time_period, mapping = sav_mappings) + df = process_mse(cohort[["person_id","index_date"]], version, window=time_period, columns = sav_columns) """ @@ -48,7 +49,7 @@ def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname raise ValueError("mapping contain no values") if columns is not None: raise ValueError("two types of mapping files given, please use only one") - mapping_dictionary = jsonref.load(open('/Users/chanminmin/Desktop/function_project/mapping_dictionary.json')) + mapping_dictionary = jsonref.load(open('../config/sav_mapping_dictionary.json')) for mapping_name in mapping: if mapping_name not in list(mapping_dictionary.keys()): raise ValueError("mapping contains invalid SAV category:subcategory names") @@ -63,10 +64,10 @@ def process_mse(cohort, version, window=None, columns=None, mapping=None, dbname if df['person_id'].nunique() != cohort['person_id'].nunique(): warnings.warn("some person_ids has no sav labels") - if window is None: - raise TypeError("window period has to be specified") - else: - df = df[(df['measurement_date'] - df['index_date']).dt.days >= window[0]] - df = df[(df['measurement_date'] - df['index_date']).dt.days <= window[1]] + if window is not None: + if len(window)!=2: + raise TypeError("more than 2 args for time window") + df = df[(df['measurement_date'] - df[reference_col]).dt.days >= window[0]] + df = df[(df['measurement_date'] - df[reference_col]).dt.days <= window[1]] return df