Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Minmin/process mse #3

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Minmin/process mse #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4873cbf
initial code for process mse
minminchan Mar 20, 2023
0f23379
add config for sav_mapping_dictionary
minminchan Mar 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 243 additions & 0 deletions config/sav_mapping_dictionary.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
{
"Abnormal or psychotic thoughts:no issues": "sav_0_aopt_0",
"Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (grandure)": "sav_1_aopt_1",
"Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (persecution)": "sav_2_aopt_2",
"Abnormal or psychotic thoughts:experiencing delusions/abnromal thoughts (religious)": "sav_3_aopt_3",
"Abnormal or psychotic thoughts:experiencing hallucinations (auditory)": "sav_4_aopt_4",
"Abnormal or psychotic thoughts:experiencing hallucinations (visual)": "sav_5_aopt_5",
"Abnormal or psychotic thoughts:experiencing hallucinations (tactile)": "sav_6_aopt_6",
"Abnormal or psychotic thoughts:History of delusions/abnormal thoughts": "sav_7_aopt_7",
"Abnormal or psychotic thoughts:History of hallucinations": "sav_8_aopt_8",
"Abnormal or psychotic thoughts:neutral/unable to categorize": "sav_9_aopt_9",
"Abnormal or psychotic thoughts:experiencing delusions (not specified)": "sav_10_aopt_10",
"Abnormal or psychotic thoughts:experiencing hallucinations (not specified)": "sav_11_aopt_11",
"Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (obsessions)": "sav_12_aopt_12",
"Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (paranoia)": "sav_13_aopt_13",
"Abnormal or psychotic thoughts:experiencing hallucinations (olfactory)": "sav_14_aopt_14",
"Abnormal or psychotic thoughts:responding to internal stimuli": "sav_15_aopt_15",
"Abnormal or psychotic thoughts:experiencing delusions/abnormal thoughts (sexual)": "sav_16_aopt_16",
"Affect:no issues/appropriate affect": "sav_17_affect_0",
"Affect:neutral/unable to categorize": "sav_18_affect_1",
"Affect:aggressive": "sav_19_affect_2",
"Affect:reactive": "sav_20_affect_3",
"Affect:irritable/angry": "sav_21_affect_4",
"Affect:improved from last visit": "sav_22_affect_5",
"Affect:issues with cognition": "sav_23_affect_6",
"Affect:loss of energy/tiredness": "sav_24_affect_7",
"Affect:blunted/restricted": "sav_25_affect_8",
"Affect:anxious/fearful": "sav_26_affect_9",
"Affect:tearful/depressed/sad": "sav_27_affect_10",
"Affect:bright/positive": "sav_28_affect_11",
"Affect:labile": "sav_29_affect_12",
"Affect:intense": "sav_30_affect_13",
"Affect:expansive/overly animated": "sav_31_affect_14",
"Affect:generally odd or out of the oridnary/inappropriate": "sav_32_affect_15",
"Affect:flooded": "sav_33_affect_16",
"Appearance:Issues with Affect": "sav_34_appearance_0",
"Appearance:Sleep/Bed": "sav_35_appearance_1",
"Appearance:Issues with Eye Contact": "sav_36_appearance_2",
"Appearance:Not age appropriate": "sav_37_appearance_3",
"Appearance:Issues with general Health": "sav_38_appearance_4",
"Appearance:Issues with Behavior": "sav_39_appearance_5",
"Appearance:Positive Affect": "sav_40_appearance_6",
"Appearance:Postivie Grooming/Hygiene": "sav_41_appearance_7",
"Appearance:Issues with Grooming/Hygiene": "sav_42_appearance_8",
"Appearance:Positive Health": "sav_43_appearance_9",
"Appearance:Issues with Weight": "sav_44_appearance_10",
"Appearance:Normal/no issues": "sav_45_appearance_11",
"Appearance:Hospital attire": "sav_46_appearance_12",
"Appearance:Positive dress": "sav_47_appearance_13",
"Appearance:Issues with dress": "sav_48_appearance_14",
"Appearance:Neutral": "sav_49_appearance_15",
"Appearance:Positive eye contact": "sav_50_appearance_16",
"Appearance:Appropriate Age": "sav_51_appearance_17",
"Appearance:Positive Behavior": "sav_52_appearance_18",
"Association:intact/no issues": "sav_53_association_0",
"Association:neutral/unable to categorize": "sav_54_association_1",
"Association:circumstantial": "sav_55_association_2",
"Association:loose": "sav_56_association_3",
"Association:tangential": "sav_57_association_4",
"Association:age appropriate": "sav_58_association_5",
"Association:improved": "sav_59_association_6",
"Association:declined": "sav_60_association_7",
"Association:disorganized": "sav_61_association_8",
"Association:general issues": "sav_62_association_9",
"Attention/Concentration:intact/no issues": "sav_63_attn_concentration_0",
"Attention/Concentration:neutral/unable to categorize": "sav_64_attn_concentration_1",
"Attention/Concentration:general issues": "sav_65_attn_concentration_2",
"Attention/Concentration:improved": "sav_66_attn_concentration_3",
"Attention/Concentration:declined": "sav_67_attn_concentration_4",
"Attention/Concentration:age appropriate": "sav_68_attn_concentration_5",
"Attention/Concentration:issues due to MH": "sav_69_attn_concentration_6",
"Attention/Concentration:issues due to DD": "sav_70_attn_concentration_7",
"Attention/Concentration:varrying": "sav_71_attn_concentration_8",
"Attitude:positive/appropriate": "sav_72_attitude_0",
"Attitude:neutral/unable to categorize": "sav_73_attitude_1",
"Attitude:age appropriate": "sav_74_attitude_2",
"Attitude:not age appropriate": "sav_75_attitude_3",
"Attitude:negative/inappropriate": "sav_76_attitude_4",
"Attitude:improved": "sav_77_attitude_5",
"Attitude:declined": "sav_78_attitude_6",
"Cognition:normal/no issues": "sav_79_cognition_0",
"Cognition:neutral/unable to categorize": "sav_80_cognition_1",
"Cognition:issues with concentration": "sav_81_cognition_2",
"Cognition:positive concentration": "sav_82_cognition_3",
"Cognition:issues with fund of knowledge": "sav_83_cognition_4",
"Cognition:positive fund of knowledge": "sav_84_cognition_5",
"Cognition:issues with attention": "sav_85_cognition_6",
"Cognition:positive attention": "sav_86_cognition_7",
"Cognition:general issues": "sav_87_cognition_8",
"Executive Functioning:in tact/noissues": "sav_88_exec_functioning_0",
"Executive Functioning:neutral/unable to categorize": "sav_89_exec_functioning_1",
"Executive Functioning:some impairment": "sav_90_exec_functioning_2",
"Executive Functioning:much impairment": "sav_91_exec_functioning_3",
"Executive Functioning:improved since last session": "sav_92_exec_functioning_4",
"Executive Functioning:declined since last session": "sav_93_exec_functioning_5",
"Executive Functioning:age appropriate": "sav_94_exec_functioning_6",
"Executive Functioning:below age level": "sav_95_exec_functioning_7",
"Executive Functioning:impaired (due to DD)": "sav_96_exec_functioning_8",
"Executive Functioning:impaired (due to TBI)": "sav_97_exec_functioning_9",
"Executive Functioning:impaired (due to other MH issue)": "sav_98_exec_functioning_10",
"Fund of knowledge:intact/no issues": "sav_99_fund_of_knowledge_0",
"Fund of knowledge:neutral/unable to categorize": "sav_100_fund_of_knowledge_1",
"Fund of knowledge:positive vocabulary": "sav_101_fund_of_knowledge_2",
"Fund of knowledge:issues with vocabulary": "sav_102_fund_of_knowledge_3",
"Fund of knowledge:positive history": "sav_103_fund_of_knowledge_4",
"Fund of knowledge:issues with history": "sav_104_fund_of_knowledge_5",
"Fund of knowledge:average": "sav_105_fund_of_knowledge_6",
"Fund of knowledge:above average": "sav_106_fund_of_knowledge_7",
"Fund of knowledge:below average": "sav_107_fund_of_knowledge_8",
"Fund of knowledge:positive current events": "sav_108_fund_of_knowledge_9",
"Fund of knowledge:issues with current events": "sav_109_fund_of_knowledge_10",
"Fund of knowledge:generally limited/issues": "sav_110_fund_of_knowledge_11",
"Fund of knowledge:declined": "sav_111_fund_of_knowledge_12",
"Fund of knowledge:improved": "sav_112_fund_of_knowledge_13",
"Gait and station:normal/no issues": "sav_113_gait_and_station_0",
"Gait and station:neutral/unable to categorize": "sav_114_gait_and_station_1",
"Gait and station:non-ambulatory": "sav_115_gait_and_station_2",
"Gait and station:issues with gait/station": "sav_116_gait_and_station_3",
"Gait and station:assistive device needed": "sav_117_gait_and_station_4",
"Homicidal:not present": "sav_118_homicidal_0",
"Homicidal:neutral": "sav_119_homicidal_1",
"Homicidal:present": "sav_120_homicidal_2",
"Impulse control:good/no issues": "sav_121_impulse_control_0",
"Impulse control:neutral/unable to categorize": "sav_122_impulse_control_1",
"Impulse control:limited/some issues": "sav_123_impulse_control_2",
"Impulse control:poor/serious issues": "sav_124_impulse_control_3",
"Impulse control:age appropriate": "sav_125_impulse_control_4",
"Impulse control:not age appropriate": "sav_126_impulse_control_5",
"Insight:no issues with insight": "sav_127_insight_0",
"Insight:issues with insight": "sav_128_insight_1",
"Insight:neutral/unable to categorize": "sav_129_insight_2",
"Intelligence:normal/no issues with intelligence": "sav_130_intelligence_0",
"Intelligence:issues with intelligence": "sav_131_intelligence_1",
"Intelligence:neutral/unable to categorize": "sav_132_intelligence_2",
"Judgment:no issues with judgement": "sav_133_judgement_0",
"Judgment:issues with judgement": "sav_134_judgement_1",
"Judgment:neutral/unable to categorize": "sav_135_judgement_2",
"Language:intact": "sav_136_language_0",
"Language:neutral/unable to categorize": "sav_137_language_1",
"Language:repetition intact": "sav_138_language_2",
"Language:issues with repetition": "sav_139_language_3",
"Language:object naming intact": "sav_140_language_4",
"Language:issues with object naming": "sav_141_language_5",
"Language:impaired": "sav_142_language_6",
"Language:non-verbal/mute": "sav_143_language_7",
"Language:minimally verbal": "sav_144_language_8",
"Language:issues related to DD": "sav_145_language_9",
"Level of consciousness:normal; no issues notes": "sav_146_conciousness_level_0",
"Level of consciousness:neutral; unable to categorize": "sav_147_conciousness_level_1",
"Level of consciousness:sleepy/drowsy": "sav_148_conciousness_level_2",
"Level of consciousness:clouded": "sav_149_conciousness_level_3",
"Level of consciousness:issues verbally": "sav_150_conciousness_level_4",
"Level of consciousness:misc. issues": "sav_151_conciousness_level_5",
"Level of consciousness:impaired due to DD/MI": "sav_152_conciousness_level_6",
"Memory:no issues/memory unimpaired": "sav_153_memory_0",
"Memory:issues/memory impaired": "sav_154_memory_1",
"Memory:neutral/unable to categorize": "sav_155_memory_2",
"Mood:Normal; no issues": "sav_156_mood_0",
"Mood:neutral; unable to categorize": "sav_157_mood_1",
"Mood:\"anxious, tense\"": "sav_158_mood_2",
"Mood:\"depressed, sad, despondent\"": "sav_159_mood_3",
"Mood:\"irritable, angry\"": "sav_160_mood_4",
"Mood:improved": "sav_161_mood_5",
"Mood:declined": "sav_162_mood_6",
"Mood:\"elevated, manic\"": "sav_163_mood_7",
"Mood:labile": "sav_164_mood_8",
"Mood:misc. issues": "sav_165_mood_9",
"Mood:suicidal": "sav_166_mood_10",
"Orientation:no orientation issues": "sav_167_orientation_0",
"Orientation:issues with orientation to person": "sav_168_orientation_1",
"Orientation:issues with orientation to place": "sav_169_orientation_2",
"Orientation:issues with orientation to time/date": "sav_170_orientation_3",
"Orientation:issues with orientation to situation": "sav_171_orientation_4",
"Orientation:neutral/unable to categorize": "sav_172_orientation_5",
"Orientation:general orientation issues": "sav_173_orientation_6",
"Psychomotor:normal; no issues": "sav_174_psychomotor_0",
"Psychomotor:neutral; unable to categorize": "sav_175_psychomotor_1",
"Psychomotor:retarded; slowed": "sav_176_psychomotor_2",
"Psychomotor:agitated; tense; restless": "sav_177_psychomotor_3",
"Psychomotor:improved": "sav_178_psychomotor_4",
"Psychomotor:declined": "sav_179_psychomotor_5",
"Psychomotor:constricted": "sav_180_psychomotor_6",
"Psychomotor:tremors": "sav_181_psychomotor_7",
"Psychomotor:tics": "sav_182_psychomotor_8",
"Psychomotor:involuntary movements": "sav_183_psychomotor_9",
"Psychomotor:misc. issues": "sav_184_psychomotor_10",
"Psychomotor:catatonic": "sav_185_psychomotor_11",
"Reasoning:normal; no issues": "sav_186_reasoning_0",
"Reasoning:neutral; unable to categorize": "sav_187_reasoning_1",
"Reasoning:impaired; poor": "sav_188_reasoning_2",
"Reasoning:impaired due to DD/cognitive issues": "sav_189_reasoning_3",
"Reasoning:impaired due to MI": "sav_190_reasoning_4",
"Reasoning:concrete": "sav_191_reasoning_5",
"Reasoning:abstract": "sav_192_reasoning_6",
"Reasoning:improved": "sav_193_reasoning_7",
"Reasoning:misc. issues": "sav_194_reasoning_8",
"Sensorium:normal; no issues": "sav_195_sensorium_0",
"Sensorium:neutral; unable to categorize": "sav_196_sensorium_1",
"Sensorium:clouded; sedate": "sav_197_sensorium_2",
"Sensorium:impaired": "sav_198_sensorium_3",
"Sensorium:fluctuating": "sav_199_sensorium_4",
"Sensorium:impaired due to medications/drugs/alcohol": "sav_200_sensorium_5",
"Sensorium:sleepy; lethargic": "sav_201_sensorium_6",
"Sleep:\"normal, no issues\"": "sav_202_sleep_0",
"Sleep:\"neutral, unable to categorize\"": "sav_203_sleep_1",
"Sleep:maintenance issues (can't stay asleep)": "sav_204_sleep_2",
"Sleep:onset issues (unable to fall asleep)": "sav_205_sleep_3",
"Sleep:nightmares": "sav_206_sleep_4",
"Sleep:insomnia": "sav_207_sleep_5",
"Sleep:hypersomnia": "sav_208_sleep_6",
"Sleep:misc. issues related to sleep": "sav_209_sleep_7",
"Speech:\"normal, no issues\"": "sav_210_speech_0",
"Speech:\"neutral, unable to categorize\"": "sav_211_speech_1",
"Speech:issues with rate": "sav_212_speech_2",
"Speech:issues with tone": "sav_213_speech_3",
"Speech:issues with volume": "sav_214_speech_4",
"Speech:stutter or lisp": "sav_215_speech_5",
"Speech:misc. issues": "sav_216_speech_6",
"Speech:impoverished": "sav_217_speech_7",
"Speech:improved": "sav_218_speech_8",
"Speech:unintelligible": "sav_219_speech_9",
"Speech:\"mute, non-verbal\"": "sav_220_speech_10",
"Suicidal:\"normal, no issues, not present\"": "sav_221_suicidal_0",
"Suicidal:\"neutral, unable to categorize\"": "sav_222_suicidal_1",
"Suicidal:suicidal ideation": "sav_223_suicidal_2",
"Suicidal:suicidal with intent": "sav_224_suicidal_3",
"Suicidal:suicidal with plan": "sav_225_suicidal_4",
"Suicidal:suicide attempt": "sav_226_suicidal_5",
"Suicidal:history of ideation": "sav_227_suicidal_6",
"Suicidal:history of attempt": "sav_228_suicidal_7",
"Suicidal:self-injurious": "sav_229_suicidal_8",
"Suicidal:history of self injury": "sav_230_suicidal_9",
"Suicidal:suicidal ideation with means": "sav_231_suicidal_10",
"Violent Thoughts:\"normal, no issues\"": "sav_232_violent_thoughts_0",
"Violent Thoughts:\"neutral, unable to categorize\"": "sav_233_violent_thoughts_1",
"Violent Thoughts:present": "sav_234_violent_thoughts_2",
"Violent Thoughts:\"present, with intent\"": "sav_235_violent_thoughts_3",
"Violent Thoughts:present with plan": "sav_236_violent_thoughts_4",
"Violent Thoughts:present with means": "sav_237_violent_thoughts_5",
"Violent Thoughts:history of violent thoughts": "sav_238_violent_thoughts_6",
"Violent Thoughts:history of violent behaviors": "sav_239_violent_thoughts_7",
"Violent Thoughts:violent/aggressive behavior": "sav_240_violent_thoughts_8"
}
73 changes: 73 additions & 0 deletions src/process_mse/process_mse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import neuroblu_postgres as nb
import jsonref
import warnings


def process_mse(cohort, version, window=None, columns=None, mapping=None, reference_col = 'index_date', dbname=None):
"""
process mse data of patients
This function returns the mse data of specified patients.
Parameters
----------
cohort (pd.DataFrame):
Minimum columns: person_id, index_date
The version of the schema as defined in OMOP CDM
version : CDM version
window : window period to filter MSE data
time window specified as a list [left cut-off , right cut-off] in days from the index date
if None, the function will return data for all SAV measurements
columns : `str, array-like of str`, *optional*
SAVs of interest specified as str or array-like collection
of str.
If None, the function will return data for all SAV columns.
mapping : `str, array-like of str`, *optional*
SAV mappings of interest specified as a list of string/s.
mapping has to be specified with two degrees of freedom separated with the semicolon delimiter
in maincategory:subcategory format (i.e. Suicidal:suicidal_ideation)
Please see list of SAV main and subcategories in Notion Page:
https://www.notion.so/holmusk/SAV-3f7fe1348c044f599c3af27c4330582a
If None, the function will return data for all SAV columns.
dbname: `string`, *optional*
The name of the database as defined in ``db.json`` file.
If None, the defaultDB is used.

Returns
A dataframe of SAV labels with measurement dates within the specified time window

Example:
time_window = [-14,365] # in days
sav_mappings = ['Suicidal:"normal, no issues, not present"', 'Suicidal:"neutral, unable to categorize"', 'Suicidal:suicidal ideation']
sav_columns = ["sav_221_suicidal_0","sav_222_suicidal_1","sav_223_suicidal_2"]

df = process_mse(cohort[["person_id","index_date"]], version, window=time_period, mapping = sav_mappings)
df = process_mse(cohort[["person_id","index_date"]], version, window=time_period, columns = sav_columns)

"""

if mapping is not None:
if len(mapping) == 0:
raise ValueError("mapping contain no values")
if columns is not None:
raise ValueError("two types of mapping files given, please use only one")
mapping_dictionary = jsonref.load(open('../config/sav_mapping_dictionary.json'))
for mapping_name in mapping:
if mapping_name not in list(mapping_dictionary.keys()):
raise ValueError("mapping contains invalid SAV category:subcategory names")
columns = list(map(mapping_dictionary.get, mapping) )

df = nb.get_SAV(cohort['person_id'].tolist(), version, columns=columns, dbname=dbname)
if df.empty:
if len(df) == 0:
raise ValueError("no person ids with sav labels")
else:
df = cohort.merge(df,on='person_id')
if df['person_id'].nunique() != cohort['person_id'].nunique():
warnings.warn("some person_ids has no sav labels")

if window is not None:
if len(window)!=2:
raise TypeError("more than 2 args for time window")
df = df[(df['measurement_date'] - df[reference_col]).dt.days >= window[0]]
df = df[(df['measurement_date'] - df[reference_col]).dt.days <= window[1]]

return df