diff --git a/docs/_build/.doctrees/environment.pickle b/docs/_build/.doctrees/environment.pickle index 7c698e5..7435204 100644 Binary files a/docs/_build/.doctrees/environment.pickle and b/docs/_build/.doctrees/environment.pickle differ diff --git a/docs/_build/genindex.html b/docs/_build/genindex.html index 266f896..9bd1d18 100644 --- a/docs/_build/genindex.html +++ b/docs/_build/genindex.html @@ -37,6 +37,7 @@

Index

C + | E | F | G | I @@ -55,6 +56,20 @@

C

+

E

+ + + +
+

F

- + + + @@ -112,14 +113,17 @@ results. - + +
    diff --git a/docs/_build/objects.inv b/docs/_build/objects.inv index f0411be..29f98ec 100644 Binary files a/docs/_build/objects.inv and b/docs/_build/objects.inv differ diff --git a/docs/_build/searchindex.js b/docs/_build/searchindex.js index 7c55d8f..9fe9d4e 100644 --- a/docs/_build/searchindex.js +++ b/docs/_build/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["about","docs","generated/mlconfound","index","install","quickstart"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,nbsphinx:3,sphinx:56},filenames:["about.md","docs.md","generated/mlconfound.rst","index.rst","install.md","quickstart.rst"],objects:{"":{mlconfound:[2,0,0,"-"]},"mlconfound.plot":{plot_graph:[1,1,1,""],plot_null_dist:[1,1,1,""]},"mlconfound.simulate":{identity:[1,1,1,""],simulate_y_c_yhat:[1,1,1,""],sinh_arcsinh:[1,1,1,""]},"mlconfound.stats":{CptResults:[1,2,1,""],ResultsFullyConfounded:[1,2,1,""],ResultsPartiallyConfounded:[1,2,1,""],full_confound_test:[1,1,1,""],generalization_test:[1,1,1,""],partial_confound_test:[1,1,1,""]},"mlconfound.stats.CptResults":{null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_x_y:[1,3,1,""],r2_x_z:[1,3,1,""],r2_y_z:[1,3,1,""]},"mlconfound.stats.ResultsFullyConfounded":{null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_y_c:[1,3,1,""],r2_y_yhat:[1,3,1,""],r2_yhat_c:[1,3,1,""]},"mlconfound.stats.ResultsPartiallyConfounded":{null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_y_c:[1,3,1,""],r2_y_yhat:[1,3,1,""],r2_yhat_c:[1,3,1,""]},mlconfound:{plot:[1,0,0,"-"],simulate:[1,0,0,"-"],stats:[1,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:function","2":"py:class","3":"py:attribute"},terms:{"0":[1,4,5],"00":5,"00814":[0,3,5],"01":5,"017867":5,"027":5,"03":5,"031732":5,"03758519":1,"039042":5,"03955979":1,"03998411":1,"067903":5,"09613765":1,"1":[0,1,4,5],"100":[1,5],"1000":[1,5],"11":4,"17":4,"175":[0,1],"187028":5,"197":[0,1],"2":[0,1,4,5],"2009":1,"2020":[0,1],"2021":[0,3,5],"210914":5,"2111":[0,3,5],"22579272":1,"237854":5,"3":1,"30471708":1,"31057339":1,"32193037":1,"323":5,"35013318":1,"4":[1,4],"42":[1,5],"48801839":1,"5":[1,5],"50":[1,5],"50886059":1,"58it":5,"595":5,"6":1,"61":4,"64it":5,"7":1,"732287":5,"7504512":1,"76":5,"761":1,"770":1,"780":1,"786173":5,"82":[0,1],"83888754":1,"88854985":1,"8900947":1,"9":1,"95":1,"96":1,"case":5,"class":1,"default":[1,5],"export":0,"float":1,"function":[1,5],"import":3,"int":1,"null":[0,1,5],"return":[1,5],"true":[1,5],"while":1,A:[1,5],As:0,For:5,In:5,No:5,The:[0,1,5],To:1,_:[0,1],ab:0,about:[1,3,4],accord:5,accur:5,across:5,addit:1,adjust:5,affect:1,ag:5,against:5,al:1,alia:1,all:[1,5],allow:1,also:5,altern:5,an:[1,5],analyz:3,appli:5,applic:1,approach:1,ar:[0,1,5],arcsinh:1,argumnet:1,array_lik:1,arxiv:[0,3,5],associ:5,assumpt:0,attribut:1,avail:1,ax:1,b:[0,1],back:[0,1,4],bar:1,barber:0,base:[1,5],batch:1,being:5,berrett:[0,1],between:[1,5],bia:[0,1,3,5],bias:[0,5],bin_c:1,bin_i:1,bin_yhat:1,binomi:[1,5],biometrika:1,bool:1,build:0,built:5,c:[1,3,5],c_i:1,c_name:[1,5],callabl:1,can:[1,5],carlo:[1,5],cat_c:1,cat_i:1,cat_yhat:1,categor:1,center:1,chain:[1,5],chang:1,character:5,ci:5,classif:1,coeffici:1,com:[3,4],comparison:5,cond_dist_method:1,condit:[0,1,5],condition:5,confid:[1,5],confoudn:[0,1],confound:[0,3],confound_test_result:1,contain:1,contribut:[1,5],control:[0,1],core:4,cornel:0,correct:5,correl:1,could:5,counfound:3,cptresult:1,creat:5,custom:5,data:[1,5],datafram:5,dataset:5,de:0,decent:5,delta:1,depend:[1,5],depict:1,describ:1,detail:[1,3],determin:1,develop:0,dict:1,direct:5,directli:[1,5],discoveri:5,displai:0,distribut:[0,1,5],document:[0,4],doe:5,dot2tex:4,dot:1,driven:[1,5],e:[1,5],edu:0,effect:[0,1,3],entir:1,epsilon:1,especi:5,essen:0,estim:[1,5],et:1,evid:5,exampl:[1,5],expect:1,explain:[1,5],explicit:5,extens:1,extrem:5,f:[0,1],fals:[1,5],fdr:5,feedback:[0,1,4],field:[0,1],figur:1,file:1,flag:[1,5],follow:5,form:5,framework:0,from:[1,3,5],full:[0,5],full_confound_test:1,fulli:[0,1,5],g:[1,5],gam:1,gener:1,generaliz:0,generalization_test:1,germani:0,git:4,github:[0,4],give:[0,1,4],given:[1,5],graph:[1,5],graphic:5,graphviz:[1,4],h0:1,h0_c:5,h0_y:5,h0_yhat:5,h1_c:5,h1_y:5,h1_yhat:5,hamper:0,handl:1,hat:[1,5],have:5,here:5,high:0,histogram:[1,5],histplot:1,hood:1,hospit:0,howev:5,http:[0,3,4],hypothesi:[0,1,5],i:[1,5],ident:1,implement:0,impli:5,increas:5,independ:[0,1,5],independencewhil:0,index:1,indic:5,input:1,instal:0,interv:1,introduc:1,investig:5,involv:1,ipynb:1,iq:5,issu:[0,4],its:1,j:0,job:1,joblib:4,jone:1,journal:[0,1],kurtosi:1,kwarg:1,lab:[0,3,4],label:5,lack:0,lb:0,learn:[1,3,5],let:5,librari:0,limit:5,line:5,linear:1,logit:1,low:[1,5],lower:5,m:1,machin:[1,3,5],mailto:0,main:[0,1,4],mani:[0,5],markov:[1,5],mathcal:1,matplotlib:1,mcmc_step:1,mean:1,method:1,methodolog:[0,1],might:5,mlconfound:[4,5],model:[0,1,3,5],modul:3,mont:[1,5],more:[1,3,5],multinomi:1,multipl:5,must:[1,5],n:[1,5],n_job:[1,5],name:[1,5],namedtupl:1,ndarrai:1,necessarili:5,next:5,non:[0,1],none:1,nonlin_trf_fun:1,normal:[0,1],note:[1,5],notebook:1,now:5,null_distribut:1,num_perm:[1,5],number:[1,5],numer:1,numpi:1,object:1,observ:[1,5],often:[0,5],onli:1,option:[4,5],out:1,outfile_bas:[1,5],output:[0,1],p:[1,5],p_ci:[1,5],packag:0,page:[0,1,3,4],panda:5,parallel:1,param:1,paramet:1,parametr:[0,5],partial:0,partial_confound_test:[1,3,5],pass:1,path:1,pd:5,perform:[1,5],permut:[0,1,5],pewsei:1,pip:[3,4],plot:[4,5],plot_graph:[1,5],plot_null_dist:[1,5],pni:[0,3,4],posit:1,power:0,precis:1,predict:[0,1,3,5],prep:1,preprint:[0,3,5],pretti:5,print:[1,5],probabl:1,process:5,processor:1,progress:1,propos:[0,5],provid:5,purpos:[1,5],put:5,py:1,pypi:3,quantif:[0,3,5],quantifi:3,quickstart:[0,1,3,4],r2:5,r2_x_y:1,r2_x_z:1,r2_y_c:1,r2_y_yhat:1,r2_y_z:1,r2_yhat_c:1,r:[0,1,5],random:[1,5],random_st:[1,5],rate:5,recent:0,recommend:1,red:5,refer:1,relat:1,represent:5,reproduc:5,research:[0,5],respect:[0,5],result:[1,5],resultsfullyconfound:1,resultspartiallyconfound:1,ret:5,return_null_dist:[1,5],rigor:0,robust:0,royal:[0,1],s:[1,5],sampl:1,samworth:0,save:1,scale:1,seaborn:[1,4,5],search:[1,3],see:1,seed:5,seri:[0,1],set:[0,1,5],sex:1,should:1,show:5,shown:0,signal:1,signific:[1,5],significantli:0,sim:1,simpli:[1,5],simul:0,simulate_y_c_yhat:[1,5],sinh:1,sinh_archsin:1,sinh_arcsinh:1,skew:1,sn:5,so:5,societi:[0,1],sole:5,some:5,sourc:1,spisak:[0,1,3,5],squar:1,stat:[3,5],state:1,statist:[0,1,3,5],statsmodel:4,step:[1,5],still:5,str:1,stronger:5,t:0,tama:[0,1],target:[1,5],techniqu:5,teh:1,test:0,test_fully_confound:1,test_partially_confound:1,than:[1,5],thei:5,theoret:0,theori:0,therefor:1,thi:[1,5],third:1,thoma:1,tool:3,tqdm:4,transform:1,transfrom:1,tupl:1,two:5,uk:0,uncondit:5,undepend:1,under:1,unit:1,univers:0,unpermut:5,upper:5,us:[1,5],valid:[0,1],valu:[1,5],variabl:[1,3,5],varianc:1,w_:1,w_cyhat:[1,5],w_yc:[1,5],w_yyhat:[1,5],wang:0,we:5,weight:1,what:5,whether:[1,5],without:1,x:1,y:[0,1,3,5],y_i:1,y_name:[1,5],yhat:[1,3],yhat_nam:[1,5],zero:[1,5]},titles:["About mlconfound","Documentation ","mlconfound","Welcome to the docs of \u2018mlconfound\u2019! ","Install","Quickstart"],titleterms:{"import":5,about:0,also:[0,4],author:0,bleed:4,bug:[0,4],citat:3,confound:[1,5],contact:[0,4],content:1,doc:3,document:[1,3],edg:4,exampl:3,from:4,full:1,github:3,h0:5,h1:5,index:3,indic:1,instal:[3,4],mlconfound:[0,1,2,3],modul:1,navig:[1,3],necessari:5,packag:5,partial:[1,5],plot:1,pypi:4,quickstart:5,recommend:4,refer:[0,5],report:[0,4],requir:4,see:[0,4],simul:[1,5],sourc:3,stabl:4,star:3,stat:1,statu:1,test:[1,5],welcom:3}}) \ No newline at end of file +Search.setIndex({docnames:["about","docs","generated/mlconfound","index","install","quickstart"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,nbsphinx:3,sphinx:56},filenames:["about.md","docs.md","generated/mlconfound.rst","index.rst","install.md","quickstart.rst"],objects:{"":{mlconfound:[2,0,0,"-"]},"mlconfound.plot":{plot_graph:[1,1,1,""],plot_null_dist:[1,1,1,""]},"mlconfound.simulate":{identity:[1,1,1,""],simulate_y_c_yhat:[1,1,1,""],sinh_arcsinh:[1,1,1,""]},"mlconfound.stats":{CptResults:[1,2,1,""],ResultsFullyConfounded:[1,2,1,""],ResultsPartiallyConfounded:[1,2,1,""],full_confound_test:[1,1,1,""],generalization_test:[1,1,1,""],partial_confound_test:[1,1,1,""]},"mlconfound.stats.CptResults":{expected_r2_x_y:[1,3,1,""],null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_x_y:[1,3,1,""],r2_x_z:[1,3,1,""],r2_y_z:[1,3,1,""]},"mlconfound.stats.ResultsFullyConfounded":{expected_r2_y_yhat:[1,3,1,""],null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_y_c:[1,3,1,""],r2_y_yhat:[1,3,1,""],r2_yhat_c:[1,3,1,""]},"mlconfound.stats.ResultsPartiallyConfounded":{expected_r2_yhat_c:[1,3,1,""],null_distribution:[1,3,1,""],p:[1,3,1,""],p_ci:[1,3,1,""],r2_y_c:[1,3,1,""],r2_y_yhat:[1,3,1,""],r2_yhat_c:[1,3,1,""]},mlconfound:{plot:[1,0,0,"-"],simulate:[1,0,0,"-"],stats:[1,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:function","2":"py:class","3":"py:attribute"},terms:{"0":[1,4,5],"00":5,"00814":[0,3,5],"01":5,"017867":5,"024":5,"027":5,"03":5,"031732":5,"03758519":1,"039042":5,"03955979":1,"03998411":1,"04":5,"044":5,"05":1,"059":5,"065":5,"067903":5,"09613765":1,"1":[0,1,4,5],"100":[1,5],"1000":[1,5],"11":4,"17":4,"175":[0,1],"187028":5,"197":[0,1],"2":[0,1,4,5],"2009":1,"2020":[0,1],"2021":[0,3,5],"210914":5,"2111":[0,3,5],"22579272":1,"237854":5,"3":1,"30471708":1,"31057339":1,"32193037":1,"323":5,"35013318":1,"4":[1,4],"42":[1,5],"48801839":1,"5":[1,5],"50":[1,5],"50886059":1,"58it":5,"595":5,"6":1,"61":4,"64it":5,"7":1,"732287":5,"7504512":1,"76":5,"761":1,"770":1,"780":1,"786173":5,"82":[0,1],"83888754":1,"88854985":1,"8900947":1,"9":1,"95":[1,5],"96":1,"case":5,"class":1,"default":[1,5],"export":0,"float":1,"function":[1,5],"import":3,"int":1,"null":[0,1,5],"return":[1,5],"true":[1,5],"while":1,A:[1,5],As:0,For:5,In:5,No:5,The:[0,1,5],To:1,_:[0,1],ab:0,about:[1,3,4],accord:5,accur:5,across:5,addit:1,adjust:5,affect:1,ag:5,against:5,al:1,alia:1,all:[1,5],allow:1,also:5,altern:5,an:[1,5],analyz:3,appli:5,applic:1,approach:1,ar:[0,1,5],arcsinh:1,argumnet:1,array_lik:1,arxiv:[0,3,5],associ:5,assumpt:0,attribut:1,avail:1,ax:1,b:[0,1],back:[0,1,4],bar:1,barber:0,base:[1,5],batch:1,being:5,berrett:[0,1],between:[1,5],bia:[0,1,3,5],bias:[0,5],bin_c:1,bin_i:1,bin_yhat:1,binomi:[1,5],biometrika:1,bool:1,build:0,built:5,c:[1,3,5],c_i:1,c_name:[1,5],callabl:1,can:[1,5],carlo:[1,5],cat_c:1,cat_i:1,cat_yhat:1,categor:1,center:1,chain:[1,5],chang:1,character:5,ci:5,classif:1,coeffici:[1,5],com:[3,4],comparison:5,cond_dist_method:1,condit:[0,1,5],condition:5,confid:[1,5],confoudn:[0,1],confound:[0,3],confound_test_result:1,contain:1,contribut:[1,5],control:[0,1],core:4,cornel:0,correct:5,correl:1,could:5,counfound:3,cptresult:1,creat:[],custom:5,data:[1,5],datafram:[],dataset:5,de:0,decent:5,delta:1,depend:[1,5],depict:1,describ:1,detail:[1,3],determin:[1,5],develop:0,dict:1,direct:5,directli:[1,5],discoveri:5,displai:0,distribut:[0,1,5],document:[0,4],doe:5,dot2tex:4,dot:1,driven:[1,5],e:[1,5],edu:0,effect:[0,1,3],entir:1,epsilon:1,especi:5,essen:0,estim:[1,5],et:1,evid:5,exampl:[1,5],expect:[1,5],expected_r2_x_i:1,expected_r2_y_yhat:1,expected_r2_yhat_c:1,explain:[1,5],explicit:5,extens:1,extrem:5,f:[0,1],fall:5,fals:[1,5],fdr:5,feedback:[0,1,4],field:[0,1],figur:1,file:1,flag:[1,5],follow:5,form:5,framework:0,from:[1,3,5],full:[0,5],full_confound_test:1,fulli:[0,1,5],g:[1,5],gam:1,gener:1,generaliz:0,generalization_test:1,germani:0,git:4,github:[0,4],give:[0,1,4],given:[1,5],graph:[1,5],graphic:5,graphviz:[1,4],h0:1,h0_c:5,h0_y:5,h0_yhat:5,h1_c:5,h1_y:5,h1_yhat:5,hamper:0,handl:1,hat:[1,5],have:5,here:5,high:0,histogram:[1,5],histplot:1,hood:1,hospit:0,howev:5,http:[0,3,4],hypothesi:[0,1,5],i:[1,5],ident:1,implement:0,impli:5,increas:5,independ:[0,1,5],independencewhil:0,index:1,indic:5,input:1,instal:0,interv:1,introduc:1,investig:5,involv:1,ipynb:1,iq:5,issu:[0,4],its:1,j:0,job:1,joblib:4,jone:1,journal:[0,1],kurtosi:1,kwarg:1,lab:[0,3,4],label:5,lack:0,lb:0,learn:[1,3,5],let:5,librari:0,limit:5,line:5,linear:1,logit:1,low:[1,5],lower:5,m:1,machin:[1,3,5],mailto:0,main:[0,1,4],mani:[0,5],markov:[1,5],mathcal:1,matplotlib:1,mcmc_step:1,mean:1,method:1,methodolog:[0,1],might:5,mlconfound:[4,5],model:[0,1,3,5],modul:3,mont:[1,5],more:[1,3,5],multinomi:1,multipl:5,must:[1,5],n:[1,5],n_job:[1,5],name:[1,5],namedtupl:1,ndarrai:1,necessarili:5,next:5,non:[0,1],none:1,nonlin_trf_fun:1,normal:[0,1],note:[1,5],notebook:1,now:5,null_distribut:1,num_perm:[1,5],number:[1,5],numer:1,numpi:1,object:1,observ:[1,5],often:[0,5],onli:1,option:[4,5],our:5,out:1,outfile_bas:[1,5],output:[0,1],outsid:5,p:[1,5],p_ci:[1,5],packag:0,page:[0,1,3,4],panda:5,parallel:1,param:1,paramet:1,parametr:[0,5],partial:0,partial_confound_test:[1,3,5],pass:1,path:1,pd:5,percentil:5,perform:[1,5],permut:[0,1,5],pewsei:1,pip:[3,4],plot:[4,5],plot_graph:[1,5],plot_null_dist:[1,5],pni:[0,3,4],posit:1,power:0,precis:1,predict:[0,1,3,5],prep:1,preprint:[0,3,5],pretti:[],print:1,probabl:1,process:5,processor:1,progress:1,propos:[0,5],provid:5,purpos:[1,5],put:5,py:1,pypi:3,quantif:[0,3,5],quantifi:3,quantil:1,quickstart:[0,1,3,4],r2:5,r2_x_y:1,r2_x_z:1,r2_y_c:1,r2_y_yhat:1,r2_y_z:1,r2_yhat_c:1,r:[0,1,5],random:[1,5],random_st:[1,5],rang:5,rate:5,recent:0,recommend:1,red:5,refer:1,reject:5,relat:1,represent:5,reproduc:5,research:[0,5],respect:[0,5],result:[1,5],resultsfullyconfound:1,resultspartiallyconfound:1,ret:5,return_null_dist:[1,5],rigor:0,robust:0,royal:[0,1],s:[1,5],sampl:1,samworth:0,save:1,scale:1,seaborn:[1,4,5],search:[1,3],see:1,seed:5,seri:[0,1],set:[0,1,5],sex:1,should:1,show:5,shown:0,signal:1,signific:[1,5],significantli:0,sim:1,simpli:[1,5],simul:0,simulate_y_c_yhat:[1,5],sinh:1,sinh_archsin:1,sinh_arcsinh:1,skew:1,sn:5,so:5,societi:[0,1],sole:5,some:5,sourc:1,spisak:[0,1,3,5],squar:1,stat:[3,5],state:1,statist:[0,1,3,5],statsmodel:4,step:[1,5],still:5,str:1,stronger:5,t:0,tama:[0,1],target:[1,5],techniqu:5,teh:1,test:0,test_fully_confound:1,test_partially_confound:1,than:[1,5],thei:5,theoret:0,theori:0,therefor:1,thi:[1,5],third:1,thoma:1,tool:3,tqdm:4,transform:1,transfrom:1,tupl:1,two:5,uk:0,uncondit:5,uncounfoudn:5,undepend:1,under:1,unit:1,univers:0,unpermut:5,upper:5,us:[1,5],valid:[0,1],valu:[1,5],variabl:[1,3,5],varianc:1,w_:1,w_cyhat:[1,5],w_yc:[1,5],w_yyhat:[1,5],wang:0,we:5,weight:1,well:5,what:5,whether:[1,5],within:5,without:1,x:1,y:[0,1,3,5],y_i:1,y_name:[1,5],yhat:[1,3],yhat_nam:[1,5],zero:[1,5]},titles:["About mlconfound","Documentation ","mlconfound","Welcome to the docs of \u2018mlconfound\u2019! ","Install","Quickstart"],titleterms:{"import":5,about:0,also:[0,4],author:0,bleed:4,bug:[0,4],citat:3,confound:[1,5],contact:[0,4],content:1,doc:3,document:[1,3],edg:4,exampl:3,from:4,full:1,github:3,h0:5,h1:5,index:3,indic:1,instal:[3,4],mlconfound:[0,1,2,3],modul:1,navig:[1,3],necessari:5,packag:5,partial:[1,5],plot:1,pypi:4,quickstart:5,recommend:4,refer:[0,5],report:[0,4],requir:4,see:[0,4],simul:[1,5],sourc:3,stabl:4,star:3,stat:1,statu:1,test:[1,5],welcom:3}}) \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 20e111a..1e1a754 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -60,8 +60,7 @@ The random seed is set for reproducible results. The flag ``return_null_dist`` is set so that the full permutation-based null distribution is returned, e.g. for plotting purposes. -The pandas dataframe is solely created for “pretty-printing” the -results. + .. code:: ipython3 @@ -101,8 +100,10 @@ results.
ci lower ci upper R2(y,c)R2(y^,c) R2(y,y^)Expected R2(y^,c)Observed R2(y^,c)
0.732287 0.786173 0.1870280.031732 0.210914[0.024, 0.04, 0.059]0.031732
+The p-value provides no evidence to reject our null-hypothesis of uncounfoudned model. +The observed coefficient-of-determination is well within the expected range (given by 5%, 50% and 95% percentiles). Let’s use the built-in plot functions of the package ``mlconfound`` for a graphical representation of the results. @@ -198,8 +202,9 @@ confounded model. ci lower ci upper R2(y,c) - R2(y^,c) R2(y,y^) + Expected R2(y^,c) + Observed R2(y^,c) @@ -209,8 +214,10 @@ confounded model. 0.017867 0.039042 0.187028 - 0.067903 0.237854 + [0.027, 0.044, 0.065] + 0.067903 + @@ -238,6 +245,8 @@ confounded model. The low p-value provides evidence against the null hypothesis of :math:`y` being conditionally independent on :math:`c` given :math:`y` and indicates that the model predictions are biased. +The observed coefficient-of-determination falls outside of the expected range (given by 5%, 50% and 95% percentiles). + +-----------------------------------------------------------------------+ | Note | diff --git a/mlconfound/stats.py b/mlconfound/stats.py index 1210f40..edae046 100755 --- a/mlconfound/stats.py +++ b/mlconfound/stats.py @@ -206,8 +206,9 @@ def _generate_X_CPT_MC(nstep, log_lik_mat, Pi, random_state=None): CptResults = namedtuple('CptResults', ['r2_x_z', - 'r2_x_y', 'r2_y_z', + 'r2_x_y', + 'expected_r2_x_y', 'p', 'p_ci', 'null_distribution']) @@ -238,25 +239,28 @@ def workhorse(_random_state): with tqdm_joblib(tqdm(desc='Permuting', total=num_perms, disable=not progress)): r2_xpi_y = np.array(Parallel(n_jobs=n_jobs)(delayed(workhorse)(i) for i in random_sates)) + expected_x_y = np.quantile(r2_xpi_y, (0.05, 0.5, 0.95)) p = np.sum(r2_xpi_y >= r2_x_y) / len(r2_xpi_y) - ci = _binom_ci(len(r2_xpi_y) * p, len(r2_xpi_y)) + ci_p = _binom_ci(len(r2_xpi_y) * p, len(r2_xpi_y)) if not return_null_dist: r2_xpi_y = None return CptResults( r2_x_z, - r2_x_y, r2_y_z, + r2_x_y, + expected_x_y, p, - ci, + ci_p, r2_xpi_y ) ResultsFullyConfounded = namedtuple('ResultsFullyConfounded', ['r2_y_c', - 'r2_y_yhat', 'r2_yhat_c', + 'r2_y_yhat', + 'expected_r2_y_yhat', 'p', 'p_ci', 'null_distribution']) @@ -273,7 +277,7 @@ def full_confound_test(y, yhat, c, num_perms=1000, Notes ----- - Performs the 'full confounder test', a statistical test described in [1]_, based + Performs the 'full confounder test', a statistical test described in [1]_, based on the conditional permutation test for independence [2]_, using a linear or a general additive model (for numerical y, based on the parameter `cond_dist_method`) and a multinomial logit model (for categorical y, undependent on cond_dist_method) to estimate the y|c conditional distribution. @@ -320,9 +324,11 @@ def full_confound_test(y, yhat, c, num_perms=1000, - "r2_y_c": coefficient-of-determination between y and c, + - "r2_yhat_c": coefficient-of-determination between yhat and c, + - "r2_y_yhat": coefficient-of-determination between y and yhat, - - "r2_yhat_c": coefficient-of-determination between yhat and c, + - "expected_r2_y_yhat": expected (0.05, 0.5, 0.95) quantiles of the coefficient-of-determination between y and yhat (given c), - "p": p-value, @@ -367,8 +373,9 @@ def full_confound_test(y, yhat, c, num_perms=1000, ResultsPartiallyConfounded = namedtuple('ResultsPartiallyConfounded', ['r2_y_c', - 'r2_yhat_c', 'r2_y_yhat', + 'r2_yhat_c', + 'expected_r2_yhat_c', 'p', 'p_ci', 'null_distribution']) @@ -432,9 +439,12 @@ def partial_confound_test(y, yhat, c, num_perms=1000, - "r2_y_c": coefficient-of-determination between y and c, + - "r2_y_yhat": coefficient-of-determination between y and yhat, + - "r2_yhat_c": coefficient-of-determination between yhat and c, - - "r2_y_yhat": coefficient-of-determination between y and yhat, + - "expected_r2_yhat_c": expected (0.05, 0.5, 0.95) quantiles of the coefficient-of-determination between yhat + and c (given y), - "p": p-value, diff --git a/notebooks/quickstart.ipynb b/notebooks/quickstart.ipynb old mode 100755 new mode 100644 index a519bca..f703943 --- a/notebooks/quickstart.ipynb +++ b/notebooks/quickstart.ipynb @@ -33,18 +33,22 @@ "##### Import the necessary packages" ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "outputs": [], "source": [ "from mlconfound.stats import partial_confound_test\n", "from mlconfound.simulate import simulate_y_c_yhat\n", "from mlconfound.plot import plot_null_dist, plot_graph\n", "\n", + "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "sns.set_style(\"whitegrid\")" @@ -65,12 +69,15 @@ "For the H0 simulation, the direct contribution of the confounder to the predicted values (`w_cyhat`) is set to zero." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "outputs": [], "source": [ "H0_y, H0_c, H0_yhat = simulate_y_c_yhat(w_yc=0.5,\n", @@ -101,7 +108,10 @@ "The pandas dataframe is solely created for \"pretty-printing\" the results." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { @@ -112,13 +122,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "Permuting: 100%|██████████| 1000/1000 [00:03<00:00, 323.64it/s]\n" + "Permuting: 100%|██████████| 1000/1000 [00:01<00:00, 529.63it/s]\n" ] }, { "data": { - "text/plain": " p ci lower ci upper R2(y,c) R2(y^,c) R2(y,y^)\n0 0.76 0.732287 0.786173 0.187028 0.031732 0.210914", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
pci lowerci upperR2(y,c)R2(y^,c)R2(y,y^)
00.760.7322870.7861730.1870280.0317320.210914
\n
" + "text/plain": " p ci lower ci upper R2(y,c) R2(y,y^) Expected R2(y^,c) \\\n0 0.761 0.733321 0.787133 0.187028 0.210914 [0.024, 0.04, 0.059] \n\n R2(y^,c) \n0 0.031732 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
pci lowerci upperR2(y,c)R2(y,y^)Expected R2(y^,c)R2(y^,c)
00.7610.7333210.7871330.1870280.210914[0.024, 0.04, 0.059]0.031732
\n
" }, "execution_count": 9, "metadata": {}, @@ -134,8 +144,9 @@ " 'ci lower' : [ret.p_ci[0]],\n", " 'ci upper' : [ret.p_ci[1]],\n", " 'R2(y,c)' : [ret.r2_y_c],\n", - " 'R2(y^,c)' : [ret.r2_yhat_c],\n", " 'R2(y,y^)' : [ret.r2_y_yhat],\n", + " 'Expected R2(y^,c)': [np.round(ret.expected_r2_yhat_c, 3)],\n", + " 'R2(y^,c)' : [ret.r2_yhat_c]\n", "})\n" ], "metadata": { @@ -151,7 +162,10 @@ "Let's use the built-in plot functions of the package `mlconfound` for a graphical representation of the results." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { @@ -169,7 +183,7 @@ { "data": { "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiKElEQVR4nO3deVyUdeIH8M9wDEGeuDCoqyVEOukGKKaspgmiIKcY6+4WeW1oKYaam2a2ZUpLay1phyJWVuvWZogImAdiWKFpKG7u5JGZaDEUoqjjzDDD8/vD+P4kAQeY0/m8X69e+cw8x2fGwQ/P9R2ZJEkSiIiIALjYOgAREdkPlgIREQksBSIiElgKREQksBSIiEhgKRARkcBSoFvWokWL8M9//hMAsH//fowaNcrkZVNSUvDRRx8BAPLz8zF9+nSz5YqJicH+/fsBAKtXr8aTTz5ptnWvWbMGS5YsMdv6yPm42ToAkb2Lj49HfHz8TedbtGgRFAoF5s2b1+p8hYWFZsm1f/9+LFy4EKWlpeKxWbNmmWXd5Ly4p0BkJQaDwdYRiG6KpUB2LTw8HOvXr0dcXByGDBmC9PR06HQ6AEBubi7+9Kc/NZm/f//++P7779u8nc8//xxRUVEYMmQIli1bhutv9L9+O5IkISMjA2FhYRg8eDDi4uJw/PhxfPjhh9i6dSvWr1+PkJAQ8Rt7eHg4srOzERcXh+DgYBgMBoSHh+OLL74Q69fr9UhPT0dISAgmTpyIb775psXX03hITKPR4NFHH0V1dTVCQkIQEhICtVp9w+Go4uJixMTEIDQ0FCkpKfj2229Nem/JebEUyO5t27YNOTk5KC4uxrFjx5Cbm2vW9Z8/fx5z5sxBeno69u3bh759+6K8vLzZeT/77DMcPHgQ27dvx1dffYWsrCx069YNkydPRlxcHGbMmIFDhw5hzZo1YpnCwkJkZ2fj4MGDcHO78YhtcXExoqKi8OWXXyI2NhaPP/446uvrW83s5eWFdevWwdfXF4cOHcKhQ4egUCiazPPdd99hwYIFePrpp1FWVoZRo0Zh1qxZ0Ov1Yh5Lv7fkeFgKZPdSUlKgUCjQrVs3jBkzBiqVyqzrLy0tRWBgIKKiouDu7o4pU6bgN7/5TbPzurm54cqVKzh16hQkSUJAQAB8fX1vmr9nz5647bbbmn1+4MCBYtvTpk2DXq9HRUVFh19XUVERRo8ejREjRsDd3R0zZsyAVqvFoUOHmmSz5HtLjoelQHbPx8dH/NnT0xMajcas66+uroafn5+Ylslk6NmzZ7PzhoWF4aGHHsKyZcsQFhaGpUuX4vLly62uv6V1Nbp+2y4uLlAoFKiurm7DK2hedXU1evXq1WTdPXv2hFqtFo9Z+r0lx8NSIIfl6ekJrVYrpn/66ad2rcfHxwdVVVViWpIk/Pjjjy3O/8gjjyA3NxdFRUU4ffo0cnJyAFwrk+a09Hij67fd0NAAtVot9j48PT1x9epV8fz1r/Fm6/X19cUPP/wgphtf168PMxFdj6VADmvAgAE4ceIEVCoVdDodVq9e3a71jB49GidOnMCOHTtgMBjw7rvv4ueff2523iNHjqCiogL19fXw9PSEXC6Hi8u1H6MePXrg7Nmzbd7+0aNHxbY3bNgAuVyOoKAg8RoLCgpgNBpRWlqKAwcOiOV69OiBCxcu4NKlS82uNzo6Gp9++inKyspQX1+Pt956C3K5HCEhIW3OSM6DpUAOq1+/fpg9ezamTp2KcePGYciQIe1aj7e3N1599VW8/PLLGDZsGL7//nsMHjy42XmvXLmCZ555Bvfddx/GjBmDbt26YcaMGQCABx98ECdPnkRoaCgef/xxk7cfERGBoqIiDB06FFu2bMHq1avh7u4OAFiyZAlKSkoQGhqKrVu3YuzYsWK5gIAAxMTEYOzYsQgNDW1yWAgA/P398Y9//AMvvPAChg8fjpKSEqxZswZyubytbxE5ERm/ZIeIiBpxT4GIiASWAhERCSwFIiISWApERCQ49Ciphw8fhoeHh5jW6XRNpu2do+UFbpL52LFr/+/f33qBbuKWe4/tkKPlBRwvs7nz6nQ6BAcHN/ucQ5eCh4cHlEqlmFapVE2m7Z2j5QVukvmxx679f88eq+W5mVvuPbZDjpYXcLzM5s7b2nAmFjt8tHjxYoSFhSE2NlY8lpmZiaioKMTFxWH27Nmoq6sTz61duxaRkZEYP3489u7da6lYRETUCouVQlJSkrj9v9GIESNQUFCArVu34s4778TatWsBACdPnkRhYSEKCwuRk5OD559/Hkaj0VLRiIioBRYrhaFDh6Jr165NHhs5cqQYOjg4OFiM+dI45rtcLkefPn1wxx134MiRI5aKRkRELbDZOYWPP/4Y0dHRAAC1Wi3GegEAhUJxwy37zdHpdE2OjWm1Woca+tfR8gKtZ+77ywibZ+zoNd1q77E9crS8gONltmZem5TCm2++CVdXV5O+97Y1PNFsfa1m9vICALt6Tbfce2yHHC0v4HiZrXmi2eqlkJubiz179uCdd94RQ/8qFIomwwer1WoO70tEZANWvXmttLQUOTk5ePPNN+Hp6SkeDw8PR2FhIfR6PSorK3H69Gnce++91oxGRESw4J7C/Pnz8eWXX6K2thajRo1CWloasrOzodfrMW3aNABAUFAQli1bhsDAQERHR2PChAlwdXXFs88+C1dXV0tFIyKiFlisFF555ZUbHktOTm5x/sceewyPNd78RERENsGxj8jh6Q0NLT7X3Mm51uYncnYOPcwFEQDI3Vzwx+yyZp/TaDTw+uWqqEYfpIZZIxaRQ+KeAhERCSwFIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGlQEREAkuBiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4GIiASWAhERCRYrhcWLFyMsLAyxsbHisQsXLmDatGkYN24cpk2bhosXLwIAJEnC8uXLERkZibi4OBw9etRSsYiIqBUWK4WkpCTk5OQ0eSw7OxthYWHYsWMHwsLCkJ2dDQAoLS3F6dOnsWPHDrzwwgt47rnnLBWLiIhaYbFSGDp0KLp27drkseLiYiQmJgIAEhMTsWvXriaPy2QyBAcHo66uDtXV1ZaKRkRELXCz5sZqamrg6+sLAPDx8UFNTQ0AQK1Ww8/PT8zn5+cHtVot5m2JTqeDSqUS01qttsm0vXO0vEDrmftqNACAM1Z+TUqlEppftv1rDQ0NzT5nz++7o30uHC0v4HiZrZnXqqVwPZlMBplM1qF1eHh4QKlUimmVStVk2t45Wl7gJpm9vADAJq/J65dt/5pGo2n2OXt+3x3tc+FoeQHHy2zuvK0VjFWvPurRo4c4LFRdXQ1vb28AgEKhQFVVlZivqqoKCoXCmtGIiAhWLoXw8HDk5eUBAPLy8hAREdHkcUmScPjwYXTu3Pmmh46IiMj8LHb4aP78+fjyyy9RW1uLUaNGIS0tDampqUhPT8emTZvQq1cvZGVlAQBGjx6NTz/9FJGRkfD09ERGRoalYhERUSssVgqvvPJKs49v2LDhhsdkMhn+9re/WSoKERGZiHc0ExGRwFIgp6M3NFhlGSJHZLNLUolsRe7mgj9ml7VpmQ9SwyyUhsi+cE+ByATcuyBnwT0FIhNw74KcBfcUiIhIYCkQEZHAUiAiIoGlQEREAkuBiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4EshiOLEjkejpJKFsORRYkcD/cUiIhIYCmQXeHhIyLb4uEjsis85ERkW9xTICIigaVAREQCS4FMpjc0QKlU2joGEVkQzymQyeRuLojPKoaXl1ezzz/7Yx0AYNkv5wR4rJ/I8dikFN555x189NFHkMlkuPvuu/Hiiy+iuroa8+fPx4ULFzBw4EC89NJLkMvltohHROS0rH74SK1W491338XHH3+MgoICGI1GFBYWYuXKlZg6dSp27tyJLl26YNOmTdaORkTk9GxyTsFoNEKr1cJgMECr1cLHxwf79u3D+PHjAQATJ05EcXGxLaIRETk1qx8+UigUmD59OsaMGQMPDw+MGDECAwcORJcuXeDmdi2On58f1Gr1Tdel0+mgUqnEtFarbTJt7xwtr1KpRENDAzQaTbPPG41GAGjyfEvztsacy7SU11q52vP362ifC0fLCzheZmvmtXopXLx4EcXFxSguLkbnzp3xxBNPYO/eve1al4eHR5OrYVQqlUNdHeNoeQHAxcWlxRPNrq6uANDk+ZbmbY05l9FoNM0+Z61c7fn7dbTPhaPlBRwvs7nztlYwVj989MUXX+C3v/0tvL294e7ujnHjxqG8vBx1dXUwGAwAgKqqKigUCmtHcyocToKImmP1PYVevXqhoqICV69exW233YaysjIMGjQIw4YNw/bt2xETE4PNmzcjPDzc2tGcCoeTIKLmWL0UgoKCMH78eEycOBFubm5QKpWYPHkyHnjgAcybNw9ZWVlQKpVITk62djQiIqdnk/sU5s6di7lz5zZ5rE+fPrwMlYjIxjjMBRERCSwFIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkQW0taRaDlyLdkDm4x9ROQM2joSLUehJXvAPQUiIhJYCkREJLAUiIhIYCkQEZFgUilMmTLFpMeIiMixtXr1kU6nw9WrV1FbW4uLFy9CkiQAwOXLl6FWq60SkIiIrKfVUvjggw+wYcMGVFdXIykpSZRCp06d8PDDD1slIBERWU+rpTBlyhRMmTIF7733HlJSUqyViYiIbMSkm9dSUlJQXl6Oc+fOwWg0iscTExMtlYuIiGzApFJYuHAhKisrMWDAALi6ugIAZDIZS4GI6BZjUil8/fXXKCoqgkwms3QeIiKyIZMuSQ0MDMRPP/1k6SxERGRjJu0p1NbWIiYmBvfeey/c3d3F42vWrLFYMCJnozc0QO7mAqVS2eZliMzFpFJIS0uzdA4ip9c4qqpGo4GXl5dJy3BkVTI3k0rhvvvuM+tG6+rq8Mwzz+D48eOQyWTIyMhAv379MG/ePJw7dw69e/dGVlYWunbtatbtEhFR60wqhZCQEHGSub6+HgaDAZ6enigvL2/XRlesWIH7778fq1atgl6vh1arxZo1axAWFobU1FRkZ2cjOzsbCxcubNf6iYiofUwqhUOHDok/S5KE4uJiHD58uF0bvHTpEg4cOIC///3vAAC5XA65XI7i4mK89957AK7d/5CSksJSICKysjafoZLJZBg7diw+++yzdm3w7Nmz8Pb2xuLFi5GYmIglS5ZAo9GgpqYGvr6+AAAfHx/U1NS0a/1ERNR+Ju0p7NixQ/y5oaEBX3/9NTw8PNq1QYPBgP/9739YunQpgoKCsHz5cmRnZzeZRyaTmXRPhE6ng0qlEtNarbbJtL2zZV6lUgmNRtPm5RoaGlpcrvFu9+ufb882zLlMS3ltnau1+Vt7j5tj68+8o/3cAY6X2Zp5TSqFkpIS8WdXV1f07t0bb7zxRrs26OfnBz8/PwQFBQEAoqKikJ2djR49eqC6uhq+vr6orq6Gt7f3Tdfl4eHR5PI9lUrVpsv5bM3WeU29wuV6Li4uLS7XeLf79c+3ZxvmXKalK3lsnau1+dty9REAm3/mbf05bg9Hy2zuvK0VjEml8OKLL5otjI+PD/z8/HDq1Cn4+/ujrKwMAQEBCAgIQF5eHlJTU5GXl4eIiAizbZOIiExjUilUVVXhhRdeEFcbhYaGYsmSJfDz82vXRpcuXYonn3wS9fX16NOnD1588UU0NDQgPT0dmzZtQq9evZCVldWudRMRUfuZVAqLFy9GbGwsXn31VQBAfn4+Fi9ejLfffrtdG1UqlcjNzb3h8Q0bNrRrfUREZB4mXX10/vx5TJo0CW5ubnBzc0NSUhLOnz9v6WxkIr2hwdYRiOgWYdKeQrdu3bBlyxbExsYCAAoKCtCtWzdL5qI2aBweoS04PAIRNcekPYWMjAxs27YNI0aMwMiRI7F9+3Zx8xkREd06TNpTWLVqFTIzM8VYRBcuXEBmZqZZr0oiIiLbM2lP4dixY00Gp+vWrZtD3fhBRESmMakUGhoacPHiRTF94cKFJt/VTEREtwaTDh9Nnz4dkydPRlRUFADgk08+waxZsywajIiIrM+kUkhMTMSgQYOwb98+AMBrr72Gu+66y6LBiIjI+kwqBQC46667WARERLc4frkrEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGlQEREAkuBiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFIiISbFYKRqMRiYmJmDlzJgCgsrISycnJiIyMRHp6OvR6va2iERE5LZuVwrvvvouAgAAxvXLlSkydOhU7d+5Ely5dsGnTJltFIyJyWjYphaqqKuzZswcPPvggAECSJOzbtw/jx48HAEycOBHFxcW2iEZE5NTcbLHRjIwMLFy4EFeuXAEA1NbWokuXLnBzuxbHz88ParX6puvR6XRQqVRiWqvVNpm2d+bKq1QqodFo2rxce5ZpaGhocTmj0XjDeq2Vq6VlWspr61ytzd/ae9wcW3/mHe3nDnC8zNbMa/VSKCkpgbe3NwYNGoT9+/d3aF0eHh5QKpViWqVSNZm2d+bM6+XlZZVlXFxcWlzO1dX1hvVaK1dLy2g0mmafs3Wu1uZvKXNLbP2Zd7SfO8DxMps7b2sFY/VSKC8vx+7du1FaWgqdTofLly9jxYoVqKurg8FggJubG6qqqqBQKKwdjYjI6Vn9nMKCBQtQWlqK3bt345VXXsHw4cPx8ssvY9iwYdi+fTsAYPPmzQgPD7d2NCIip2c39yksXLgQb7/9NiIjI3HhwgUkJyfbOhKR3dMbGqyyDDkPm5xobjRs2DAMGzYMANCnTx9ehkrURnI3F/wxu6xNy3yQGmahNHQrsJs9BSIisj2WAhERCSwFIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGlQEREAkuBiIgEloKd4VclEpEt2fTrOOlG/HpFIrIl7ikQEZHAUiAiIoGlQEREAkuBiIgElgIREQksBSIn09bLnnmZtHPhJalETqatlz3zkmfnYvVS+PHHH/HXv/4VNTU1kMlk+MMf/oApU6bgwoULmDdvHs6dO4fevXsjKysLXbt2tXY8IiKnZvXDR66urli0aBGKiorw4YcfYuPGjTh58iSys7MRFhaGHTt2ICwsDNnZ2daORkTk9KxeCr6+vhg4cCAAoFOnTvD394darUZxcTESExMBAImJidi1a5e1oxEROT2bnlM4e/YsVCoVgoKCUFNTA19fXwCAj48Pampqbrq8TqeDSqUS01qttsm0vWsur1KphEajafO6rLVMQ0NDi8sZjcYb1mvr19JSXlvnam3+1t5jc2yjrcvoDQ2Qu7X8+6NSqbxx/Vo9vv/u2zbnspZb4d8KS7FZKVy5cgVz587F008/jU6dOjV5TiaTQSaT3XQdHh4eTT6QKpWq2Q+ovWopr5eXV5vXZa1lXFxcWlzO1dX1hvXa+rVoNJpmn7N1rtbmbymzubbR1mVudmK6ubwfpIbZ9c/irfJvRUfW1xKbXJJaX1+PuXPnIi4uDuPGjQMA9OjRA9XV1QCA6upqeHt72yIaEZFTs3opSJKEJUuWwN/fH9OmTROPh4eHIy8vDwCQl5eHiIgIa0cjInJ6Vj989NVXX2HLli24++67kZCQAACYP38+UlNTkZ6ejk2bNqFXr17IysqydjQiIqdn9VIIDQ3FsWPHmn1uw4YNVk5DRETX4zAXREQksBSIiEhgKRARkcBSsLDWRph0pOukicg5cJRUC2vtxp+WbvohIrIV7ikQEZHAUiAiIoGlQERm155va+M3vNkHnlMgIrNr67e7ATyfZi+4p0BERAJLgYjsQlsPH/Fwk2Xw8BER2YW2HnLi4SbL4J4CEREJLAUiIhJYCkREJLAUiIhIYCm0Aa92IKJbHa8+agPekENEtzruKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFInJI/M4Gy+AlqUTkkNpzifi704cBAJRKpcnL6A0NkLs5z+/PLAUichqNRaLRaODl5WXSMs52r5Hz1B8RUTs42/c82N2eQmlpKVasWIGGhgYkJycjNTXV1pGIyIk52/c82NWegtFoxLJly5CTk4PCwkIUFBTg5MmTFtmWo7c5EdknS5wAb+4ciKX+DbOrPYUjR47gjjvuQJ8+fQAAMTExKC4uxl133WX2bXEcIyKyhPb+29LaMs2dA7HUv0cySZIki6y5HT755BPs3bsXK1asAADk5eXhyJEjePbZZ5ud//Dhw/Dw8LBmRCIih6fT6RAcHNzsc3a1p9BWLb0oIiJqH7s6p6BQKFBVVSWm1Wo1FAqFDRMRETkXuyqF3/3udzh9+jQqKyuh1+tRWFiI8PBwW8ciInIadnX4yM3NDc8++yz+8pe/wGg0YtKkSQgMDLR1LCIip2FXJ5qJiMi27OrwERER2RZLgYiIBIcohdLSUowfPx6RkZHIzs6+4Xm9Xo/09HRERkYiOTkZZ8+eBQDU1tYiJSUFISEhWLZsmUNk/vzzz5GUlIS4uDgkJSWhrKxtN8HYIvORI0eQkJCAhIQExMfHY+fOnXadt9EPP/yAkJAQrF+/3q7znj17Fvfee694j1u6b8eeMgPAN998g8mTJyMmJgZxcXHQ6XR2nTk/P1+8xwkJCRgwYABUKpXd5q2vr8dTTz2FuLg4REdHY+3ateYJJNk5g8EgRURESGfOnJF0Op0UFxcnnThxosk877//vrR06VJJkiSpoKBAeuKJJyRJkqQrV65IBw4ckDZu3Cg9//zzDpH56NGjUlVVlSRJknTs2DFp5MiRdp9Zo9FI9fX1kiRJklqtloYPHy6m7TFvo7S0NCktLU3KycmxaNaO5q2srJRiYmIsnvHXOpK5vr5eio2NlVQqlSRJknT+/HnJYDDYdebrffPNN1JERIRd583Pz5fS09MlSbr2MzhmzBipsrKyw5nsfk/h+qEv5HK5GPriert378bEiRMBAOPHj0dZWRkkSYKXlxdCQ0OtftdzRzLfc8894t6MwMBA6HQ66PV6u87s6ekJN7drF7LpdDrIZDK7zgsAu3btQu/eva12dVtH89pCRzJ//vnn6N+/PwYMGAAA6N69O1xdXe068/UKCwsRExNj13llMhmuXr0Kg8EArVYLd3d3dOrUqcOZ7L4U1Go1/Pz8xLRCoYBarb5hnp49ewK4dllr586dUVtba9Wcv85jjszbt2/HPffcA7lcbveZKyoqEBMTg/j4eDz//POiJOwx75UrV7Bu3TrMmTPHohnNlRe4dggpMTERDz/8MA4ePGj3mb/77jvIZDLMmDEDEydOxLp16+w+8/WKioqsUgodyTt+/Hh4enpi5MiRGDNmDKZPn45u3bp1OJNd3adA/+/EiRNYuXIl3nrrLVtHMUlQUBAKCwvx7bff4qmnnsKoUaPsdlyq1157DVOmTMHtt99u6ygm8fX1RUlJCbp3746vv/4as2fPRmFhoVl+K7QUo9GIr776Cps2bYKnpyemTp2KQYMGISzM/geVrKiogKenJ+6++25bR2nVkSNH4OLigr1796Kurg5//vOf8fvf/14MKNpedr+nYMrQFwqFAj/++CMAwGAw4NKlS+jevbtVc/46T0cyV1VVYc6cOcjMzETfvn0dInOjgIAAeHl54fjx43abt6KiAitXrkR4eDg2bNiAtWvX4v3337fbvHK5XLzPgwYNQt++ffHdd99ZNG9HM/v5+WHo0KHw9vaGp6cnRo0ahaNHj9p15kbWOnTU0bwFBQW4//774e7ujh49emDw4MH473//2+FMdl8Kpgx9ER4ejs2bNwO4dshl+PDhVjmu3ZKOZK6rq0NqaioWLFiAIUOGOETmyspKGAwGAMC5c+dw6tQp9O7d227zbty4Ebt378bu3bsxZcoUzJw5Ew8//LDd5j1//jyMRiMAoLKyEqdPn+7wb4OWzjxy5EgcP35cHPM+cOCARYbAN2dmAGhoaMC2bdusVgodyduzZ0/s378fwLWhtSsqKuDv79/xUB0+VW0Fe/bskcaNGydFRERIb7zxhiRJkpSVlSXt2rVLkiRJ0mq1UlpamjR27Fhp0qRJ0pkzZ8SyY8aMkYYOHSoFBwdL999//w1n9u0t8+uvvy4FBQVJ8fHx4r+ff/7ZrjNv3rxZmjBhghQfHy8lJiZKO3futOu811u1apVVrj7qSN5PPvmkyftbXFxslbwdySxJkpSXlydNmDBBiomJkTIzMx0i8759+6Tk5GSrZe1I3suXL0tpaWnShAkTpOjoaGndunVmycNhLoiISLD7w0dERGQ9LAUiIhJYCkREJLAUiIhIYCkQEZHAUiCnpVQqkZCQgNjYWMyaNQt1dXUAAJVK1WR0z6KioibLzZ07F5WVlTddv8FgQGpqKoYNG9bmm/kaLwpcvXq1mNbr9XjooYfEPSFElsBSIKd12223YcuWLSgoKEDXrl3xr3/9SzyemZmJwsJC5OTkICMjQxTGiRMnYDQaTbp57LnnnoO/vz9ef/11zJs3r8mdqzeTn5+PnJwc6HQ6rFu3Dvn5+ZDL5QgLC7uhpIjMiaVABCA4OFgMRNavXz/ceeedAK4NMeDt7Y3z588DALZu3YqIiAgAwKZNm7BixQqxjv/85z/IyMgAcG18pc6dO2PRokUIDQ3F8uXLMX/+fFy6dOmGbf/888+YPXs24uPjER8fj/LyciQkJMDPzw/r169Hr169kJCQAAAYO3Ystm7darH3gYilQE7PaDSirKzshuEFgGuDjtXX14sxqMrLyzFw4EAAQHR0NEpKSlBfXw8AyM3NxaRJkwAAc+bMwVNPPSXWExISgo0bN6Jz5843bGP58uUYOnQo8vPzsXnzZgQGBmLr1q2oqqrCjBkz8MMPP4giCAwMNMv4NkQt4Sip5LS0Wi0SEhKgVqsREBCAESNGNHm+uroaCxcuRGZmJlxcrv3+9NNPP8Hb2xsAcPvtt2P48OHYs2cP/P39UV9fj/79+7c5x759+/DSSy8BAFxdXdG5c2fExsZCJpNh9erVePTRR8U5BldXV7i7u+Py5ct2PUoqOS7uKZDTajynUFJSAkmSxDkFALh8+TJmzpyJefPmITg4WDzu4eHR5Gslk5OTkZubi9zcXCQlJZktW+MAbWlpaU2mgWtfz2ivw5KT42MpkNPz9PTEM888g7fffhsGgwF6vR6zZ89GQkICoqKimswbEBCAM2fOiOmgoCBUVVWhoKAAsbGxJm/z/fffF8N1h4WFYePGjQCuHcpq7rxDo9raWnTv3h3u7u5teYlEJmMpEAG455570L9/fxQUFGDbtm04ePAgNm/eLL7EvfEL3EePHi2GK24UHR2NwYMHo2vXriZv79SpU+JbspYsWYL9+/cjLi4OSUlJOHnyZIvL7d+/Hw888ECbXx+RqThKKlEbaLVaPPLII/j3v/8tvnN45syZmDp1apu+VWzmzJlYvXp1m79qdc6cOViwYAH69evXpuWITMVSIGqjvXv3IiAgAJ06dUJycjL69++PVatWWXy7er0eRUVFSExMtPi2yHmxFIiISOA5BSIiElgKREQksBSIiEhgKRARkcBSICIi4f8A01a9ZU4bknEAAAAASUVORK5CYII=\n" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgn0lEQVR4nO3de1hUdeI/8PfILcgL4nJR13aFTCfdAMV0VtMAryEXMdbdLcPLhn5TDDU3zWzLlNbWWlarxxBrbVvXNhZRAfMyYriFpKG4ueMtNdFkKARRx5lhZs7vD5fPD5TLDMwV3q/n2SfO4XzOeQ87+OacOReZJEkSiIiIAHRxdAAiInIeLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGlQB3WsmXL8Oc//xkAUFJSgjFjxpg9dsaMGfj0008BADt37sTs2bOtlismJgYlJSUAgA0bNuCFF16w2ro3btyIFStWWG191Pm4OzoAkbOLi4tDXFxcq8stW7YMgYGBWLRoUYvL5efnWyVXSUkJli5diqKiIjFv3rx5Vlk3dV7cUyCyE4PB4OgIRK1iKZBTi4qKwubNmxEbG4thw4YhLS0NOp0OAJCTk4Pf/OY3jZYfOHAgvvvuO4u388UXX2DSpEkYNmwYVq1ahYYX+jfcjiRJSE9Ph0KhwNChQxEbG4szZ87gk08+wa5du7B582aEh4eLv9ijoqKQmZmJ2NhYhIWFwWAwICoqCl9++aVYv16vR1paGsLDwzF16lScOnWq2ddTf0hMo9Hg2WefRWVlJcLDwxEeHg61Wn3P4SilUomYmBhERERgxowZ+Pbbb8362VLnxVIgp7d7925kZWVBqVTi9OnTyMnJser6r127hgULFiAtLQ2HDx/GAw88gNLS0iaX/fe//42jR49iz549+Prrr5GRkQFfX19Mnz4dsbGxmDNnDo4dO4aNGzeKMfn5+cjMzMTRo0fh7n7vEVulUolJkybhq6++wpQpU/Dcc8+hrq6uxcw+Pj7YtGkTAgICcOzYMRw7dgyBgYGNlrlw4QKWLFmCl156CcXFxRgzZgzmzZsHvV4vlrH1z5ZcD0uBnN6MGTMQGBgIX19fREZGQqVSWXX9RUVFGDBgACZNmgQPDw8kJyfjJz/5SZPLuru749atWzh//jwkSUJISAgCAgJazd+7d2/cd999TX5/8ODBYtuzZs2CXq9HWVlZu19XQUEBxo4di1GjRsHDwwNz5syBVqvFsWPHGmWz5c+WXA9LgZyev7+/+Nrb2xsajcaq66+srERQUJCYlslk6N27d5PLKhQKPPXUU1i1ahUUCgVWrlyJmzdvtrj+5tZVr+G2u3TpgsDAQFRWVlrwCppWWVmJPn36NFp37969oVarxTxb/2zJ9bAUyGV5e3tDq9WK6R9++KFN6/H390dFRYWYliQJV69ebXb5Z555Bjk5OSgoKMDFixeRlZUF4E6ZNKW5+fUabttkMkGtVou9D29vb9y+fVt8v+FrbG29AQEB+P7778V0/eu6+zATUUMsBXJZgwYNwtmzZ6FSqaDT6bBhw4Y2rWfs2LE4e/Ys9u7dC4PBgI8++gg//vhjk8ueOHECZWVlqKurg7e3Nzw9PdGly51fo169euHy5csWb//kyZNi21u2bIGnpydCQ0PFa8zLy4PRaERRURGOHDkixvXq1Qs1NTW4ceNGk+udPHkyPv/8cxQXF6Ourg4ffPABPD09ER4ebnFG6jxYCuSy+vfvj/nz52PmzJmYMGEChg0b1qb1+Pn54S9/+QveeustjBgxAt999x2GDh3a5LK3bt3Cyy+/jEcffRSRkZHw9fXFnDlzAABPPvkkzp07h4iICDz33HNmbz86OhoFBQUYPnw4duzYgQ0bNsDDwwMAsGLFChQWFiIiIgK7du3CuHHjxLiQkBDExMRg3LhxiIiIaHRYCACCg4Pxpz/9Ca+//jpGjhyJwsJCbNy4EZ6enpb+iKgTkfEhO0REVI97CkREJLAUiIhIYCkQEZHAUiAiIsGl75J6/PhxeHl5iWmdTtdo2tm5Wl6ghcynT9/578CB9g3Uig71M3ZSrpYXcL3M1s6r0+kQFhbW5PdcuhS8vLwgl8vFtEqlajTt7FwtL9BC5v/7vzv/PXjQrnla06F+xk7K1fICrpfZ2nlbup0JDx8REZHAUiAiIoGlQEREAkuBiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSoA5HbzCJry25CrThOKLOyqVvc0HUFE/3Lvh1ZjEAQKPRwMfHx6xx21IUtoxF5BK4p0BERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiP6nLaek8jRW6mh4SirR/zQ8ldVcPI2VOhruKRARkcBSICIigaVATo3H7Insi58pkFPjcX4i++KeAhERCSwFIiISWApERCSwFIiISLBZKSxfvhwKhQJTpkwR82pqajBr1ixMmDABs2bNwvXr1wEAkiRh9erVGD9+PGJjY3Hy5ElbxSIiohbYrBQSExORlZXVaF5mZiYUCgX27t0LhUKBzMxMAEBRUREuXryIvXv34vXXX8err75qq1hERNQCm5XC8OHD0aNHj0bzlEolEhISAAAJCQnYv39/o/kymQxhYWGora1FZWWlraIREVEz7HqdQlVVFQICAgAA/v7+qKqqAgCo1WoEBQWJ5YKCgqBWq8WyzdHpdFCpVGJaq9U2mnZ2rpYXaD7zAxoNAOCSlV+PXC6H5n/rtkT9GJPJZNH4tmzL2v8futr7wtXyAq6X2Z55HXbxmkwmg0wma9c6vLy8Gj2YXaVSWfSgdkdztbxAC5n/9xxkW7wec5+x3NQYS57R3NZtWfs1u9r7wtXyAq6X2dp5WyoYu5591KtXL3FYqLKyEn5+fgCAwMBAVFRUiOUqKioQGBhoz2hERAQ7l0JUVBRyc3MBALm5uYiOjm40X5IkHD9+HN26dWv10BEREVmfzQ4fLV68GF999RWqq6sxZswYpKamIiUlBWlpacjOzkafPn2QkZEBABg7diw+//xzjB8/Ht7e3khPT7dVLCIiaoHNSuHtt99ucv6WLVvumSeTyfCHP/zBVlGIiMhMvKKZiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCtYneYLLLGCKyL5s9jpM6Nk/3Lvh1ZrGYfuVqLQBgVYN5d9uWomg0xhzbUhRtC0hEbcI9BSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFIiISHHJF81//+ld8+umnkMlkeOihh/DGG2+gsrISixcvRk1NDQYPHow333wTnp6ejohHRNRp2X1PQa1W46OPPsK//vUv5OXlwWg0Ij8/H+vWrcPMmTOxb98+dO/eHdnZ2faORkTU6Tnk8JHRaIRWq4XBYIBWq4W/vz8OHz6MiRMnAgCmTp0KpVLpiGhEFrHFjQHlcrlVtkPUFnY/fBQYGIjZs2cjMjISXl5eGDVqFAYPHozu3bvD3f1OnKCgIKjVantHI7LY3TcGNEdrNwbUaDTw8fG5ZwyRPdi9FK5fvw6lUgmlUolu3brh+eefx6FDh9q0Lp1OB5VKJaa1Wm2jaWfnanmB/59ZLpdDo9GI+UajEQAazWtKa9+39hiTyWTReHvna0pzmZ31veLK72NXYc+8di+FL7/8Ej/96U/h5+cHAJgwYQJKS0tRW1sLg8EAd3d3VFRUIDAwsNV1eXl5NdrVrv/HylU4Q169wQRP97YdRWz416ybm9s981ob05btWDqmqb+6bbUta41pLrOj3yvNcYb3saVcLbO187ZUMHYvhT59+qCsrAy3b9/Gfffdh+LiYgwZMgQjRozAnj17EBMTg+3btyMqKsre0TolSw9/1P+DxcMZRB2T3UshNDQUEydOxNSpU+Hu7g65XI7p06fj8ccfx6JFi5CRkQG5XI6kpCR7RyMi6vQccp3CwoULsXDhwkbz+vXrx9NQiYgcjFc0ExGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGlQEREAkuhA+HTuYiovRxyQzyyjbY+BYyIqB73FIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIsGsUkhOTjZrHhERubYWb4in0+lw+/ZtVFdX4/r165AkCQBw8+ZNqNVquwQkIiL7abEUtm3bhi1btqCyshKJiYmiFLp27Yqnn37aLgGJiMh+WiyF5ORkJCcn429/+xtmzJhhr0xEROQgZj1PYcaMGSgtLcWVK1dgNBrF/ISEBFvlIiIiBzCrFJYuXYry8nIMGjQIbm5uAACZTNbmUqitrcXLL7+MM2fOQCaTIT09Hf3798eiRYtw5coV9O3bFxkZGejRo0eb1k9ERG1jVil88803KCgogEwms8pG16xZg8ceewzr16+HXq+HVqvFxo0boVAokJKSgszMTGRmZmLp0qVW2R4REZnHrFNSBwwYgB9++MEqG7xx4waOHDmCJ598EgDg6emJ7t27Q6lUij2PhIQE7N+/3yrbIyIi85m1p1BdXY2YmBg88sgj8PDwEPM3btxo8QYvX74MPz8/LF++HKdOncLgwYOxYsUKVFVVISAgAADg7++PqqqqVtel0+mgUqnEtFarbTTt7KydVy6XQ6PRWDzOkjEmk0ks33Bc/WdNra3L1vnuHtMwr623Za0xzWV21ve2q/3eAa6X2Z55zSqF1NRUq23QYDDgv//9L1auXInQ0FCsXr0amZmZjZaRyWRmHary8vKCXC4X0yqVqtG0s7NFXh8fH5uO0Wg0YvmG4+o/a2ptXbbOd/eYhnltvS1rjWkus7O+t13t9w5wvczWzttSwZhVCo8++qjVwgQFBSEoKAihoaEAgEmTJiEzMxO9evVCZWUlAgICUFlZCT8/P6ttk4iIzGNWKYSHh4u/3Ovq6mAwGODt7Y3S0lKLN+jv74+goCCcP38ewcHBKC4uRkhICEJCQpCbm4uUlBTk5uYiOjra4nUTEVH7mFUKx44dE19LkgSlUonjx4+3eaMrV67ECy+8gLq6OvTr1w9vvPEGTCYT0tLSkJ2djT59+iAjI6PN6yciorYxqxQakslkGDduHN555x288MILbdqoXC5HTk7OPfO3bNnSpvUREZF1mFUKe/fuFV+bTCZ888038PLyslkoIiJyDLNKobCwUHzt5uaGvn374r333rNZKCIicgyzSuGNN96wdQ4iInICZl3RXFFRgfnz50OhUEChUCA1NRUVFRW2zkZERHZmViksX74cUVFROHToEA4dOoTIyEgsX77c1tmIiMjOzCqFa9euYdq0aXB3d4e7uzsSExNx7do1W2cjIiI7M6sUfH19sWPHDhiNRhiNRuzYsQO+vr42jkZERPZmVimkp6dj9+7dGDVqFEaPHo09e/bgj3/8o62zERGRnZl19tH69euxdu1a8dCbmpoarF27lmclERF1MGbtKZw+fbrRU9B8fX1d6razRERkHrNKwWQy4fr162K6pqam0bOaiYioYzDr8NHs2bMxffp0TJo0CQDw2WefYd68eTYNRkRE9mdWKSQkJGDIkCE4fPgwAOCdd97Bgw8+aNNgRERkf2bfJfXBBx9kERARdXBmfaZARESdA0uBiIgElgIREQksBSIiElgKREQksBSIiEhgKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFIiISWApERCSwFIiISHBYKRiNRiQkJGDu3LkAgPLyciQlJWH8+PFIS0uDXq93VDQiok7LYaXw0UcfISQkREyvW7cOM2fOxL59+9C9e3dkZ2c7KhoRUaflkFKoqKjAwYMH8eSTTwIAJEnC4cOHMXHiRADA1KlToVQqHRGNiKhTM/vJa9aUnp6OpUuX4tatWwCA6upqdO/eHe7ud+IEBQVBrVa3uh6dTgeVSiWmtVpto2lnZ+28crkcGo3G4nGWjDGZTGL5huOMRqNZ67J1vrvHNMxr621Za0xTmfUGEzzdLf8bTqPV47sL31o8zhKu9nsHuF5me+a1eykUFhbCz88PQ4YMQUlJSbvW5eXlBblcLqZVKlWjaWdni7w+Pj42HaPRaMTyDce5ubmZtS5b57t7TMO8tt6WtcY0ldnTvQt+nVls8Xa2pShs/jvhar93gOtltnbelgrG7qVQWlqKAwcOoKioCDqdDjdv3sSaNWtQW1sLg8EAd3d3VFRUIDAw0N7RiIg6Pbt/prBkyRIUFRXhwIEDePvttzFy5Ei89dZbGDFiBPbs2QMA2L59O6KiouwdjYio03Oa6xSWLl2KDz/8EOPHj0dNTQ2SkpIcHYmIqNNxyAfN9UaMGIERI0YAAPr168fTUImIHMxp9hSIiMjxWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJLAUiIhJYCkREJLAUiIhIYCkQEZHAUiAiIoGl4KT0BpOjI1AH0Jb3Ed97nZtDb51NzWvL4xe3pShslIZcFd9HZCnuKRARkcBSICIigaVAREQCS4GIiASWAhERCSwFIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYiIBJYCEREJdr9L6tWrV/H73/8eVVVVkMlk+NWvfoXk5GTU1NRg0aJFuHLlCvr27YuMjAz06NHD3vGIiDo1u+8puLm5YdmyZSgoKMAnn3yCrVu34ty5c8jMzIRCocDevXuhUCiQmZlp72hERJ2e3UshICAAgwcPBgB07doVwcHBUKvVUCqVSEhIAAAkJCRg//799o5GRLD8ITtyuZwP5ulAHPqQncuXL0OlUiE0NBRVVVUICAgAAPj7+6OqqqrV8TqdDiqVSkxrtdpG086upbxyuRwajcbiddp6jMlkEss3HGc0Gs1al71fU8O8tt6WtcY0l7kt22nLOE/3LojLUJq9vMlkQt7i8R3md88Z2TOvw0rh1q1bWLhwIV566SV07dq10fdkMhlkMlmr6/Dy8oJcLhfTKpWq0bSzay2vj4+Pxeu09RiNRiOWbzjOzc3NrHXZ+zU1zGvrbVlrTHOZ27Kdto6z9D0BoEP97jkba+dtqWAccvZRXV0dFi5ciNjYWEyYMAEA0KtXL1RWVgIAKisr4efn54hoRESdmt1LQZIkrFixAsHBwZg1a5aYHxUVhdzcXABAbm4uoqOj7R2NiKjTs/vho6+//ho7duzAQw89hPj4eADA4sWLkZKSgrS0NGRnZ6NPnz7IyMiwdzQiok7P7qUQERGB06dPN/m9LVu22DkNERE1xCuaiYhIYCkQEZHAUiAiIoGlQEREAkvBDpq7BYArXTxD1JK23OaCt8ZwTg69zUVn4eneBb/OLL5nfktX225LUdg6FpHVNPcebwnf486JewpERCSwFIiISGApEBGRwFIgIiKBpUBERAJLwUI8jY7IOngaq3PiKakW4ql3RNbB3yXnxD0FIiISWApERCSwFIiISGApEBGRwFIgIiKBpUBERAJLgYhcRluvU+D1DebjdQpE5DLacm0DwOsbLME9BSIiElgKREQksBSIiEhgKRARkcBSICIigaVARB3e3aekyuVyi8e0ZTuuiKekElGHd/eprBqNBj4+Pi2O2Zai6JS39u60ewododGJiKzNqfYUioqKsGbNGphMJiQlJSElJcVm2+JFMERE93KaPQWj0YhVq1YhKysL+fn5yMvLw7lz5xwdi4jIpsw5atHUZyC2OtrhNHsKJ06cwM9+9jP069cPABATEwOlUokHH3zQwcmIiGzHnKMWTX0GYqujFjJJkiSbrNlCn332GQ4dOoQ1a9YAAHJzc3HixAm88sorzY45fvw4vLy87BWRiKhD0Ol0CAsLa/J7TrOn0BbNvSgiImobp/lMITAwEBUVFWJarVYjMDDQgYmIiDofpymFX/ziF7h48SLKy8uh1+uRn5+PqKgoR8ciIupUnObwkbu7O1555RX87ne/g9FoxLRp0zBgwABHxyIi6lSc5oNmIiJyPKc5fERERI7HUiAiIsElSqGoqAgTJ07E+PHjkZmZec/39Xo90tLSMH78eCQlJeHy5csAgOrqasyYMQPh4eFYtWqVS2T+4osvkJiYiNjYWCQmJqK42PJbcdgz74kTJxAfH4/4+HjExcVh3759dsnbnsz1vv/+e4SHh2Pz5s1Onffy5ct45JFHxM+5pWt3nCUzAJw6dQrTp09HTEwMYmNjodPpnDbvzp07xc83Pj4egwYNgkqlsnne9mSuq6vDiy++iNjYWEyePBnvv/++dQJJTs5gMEjR0dHSpUuXJJ1OJ8XGxkpnz55ttMzHH38srVy5UpIkScrLy5Oef/55SZIk6datW9KRI0ekrVu3Sq+99ppLZD558qRUUVEhSZIknT59Who9erRT59VoNFJdXZ0kSZKkVqulkSNHimlnzVwvNTVVSk1NlbKyspw6b3l5uRQTE2PzjHdrT+a6ujppypQpkkqlkiRJkq5duyYZDAanzdvQqVOnpOjoaJtmrdeezDt37pTS0tIkSbrzexgZGSmVl5e3O5PT7yk0vP2Fp6enuP1FQwcOHMDUqVMBABMnTkRxcTEkSYKPjw8iIiLsftVzezI//PDD4vqMAQMGQKfTQa/XO21eb29vuLvfOYlNp9NBJpPZNKs1MgPA/v370bdvX7ud4dbevI7QnsxffPEFBg4ciEGDBgEAevbsCTc3N6fN21B+fj5iYmJsmtUamWUyGW7fvg2DwQCtVgsPDw907dq13ZmcvhTUajWCgoLEdGBgINRq9T3L9O7dG8CdU1u7deuG6upqu+a8O481Mu/ZswcPP/wwPD09nTpvWVkZYmJiEBcXh9dee02UhLNmvnXrFjZt2oQFCxbYPKc18gJ3DiElJCTg6aefxtGjR50+84ULFyCTyTBnzhxMnToVmzZtcuq8DRUUFNitFNqTeeLEifD29sbo0aMRGRmJ2bNnw9fXt92ZnOY6BWrs7NmzWLduHT744ANHR2lVaGgo8vPz8e233+LFF1/EmDFjnPqeVO+88w6Sk5Nx//33OzqKWQICAlBYWIiePXvim2++wfz585Gfn2+VvwptxWg04uuvv0Z2dja8vb0xc+ZMDBkyBAqFc996vqysDN7e3njooYccHaVVJ06cQJcuXXDo0CHU1tbit7/9LX75y1+Km4q2ldPvKZhz+4vAwEBcvXoVAGAwGHDjxg307NnTrjnvztOezBUVFViwYAHWrl2LBx54wOnz1gsJCYGPjw/OnDnj1JnLysqwbt06REVFYcuWLXj//ffx8ccfO21eT09P8bMeMmQIHnjgAVy4cMGmedubOSgoCMOHD4efnx+8vb0xZswYnDx50mnz1rPnoaP6PG3NnJeXh8ceewweHh7o1asXhg4div/85z/tzuT0pWDO7S+ioqKwfft2AHcOuYwcOdJux7ab0p7MtbW1SElJwZIlSzBs2DCnz1teXg6DwQAAuHLlCs6fP4++ffs6deatW7fiwIEDOHDgAJKTkzF37lw8/fTTTpv32rVrMBqNAIDy8nJcvHix3X8N2jrz6NGjcebMGXHM+8iRIza/DX57/60wmUzYvXu3XUuhPZl79+6NkpISAHdurV1WVobg4OD2h2r3R9V2cPDgQWnChAlSdHS09N5770mSJEkZGRnS/v37JUmSJK1WK6Wmpkrjxo2Tpk2bJl26dEmMjYyMlIYPHy6FhYVJjz322D2f7Dtb5nfffVcKDQ2V4uLixP9+/PFHp827fft26YknnpDi4uKkhIQEad++fTbP2t7MDa1fv94uZx+1J+9nn33W6GesVCrtkrc9mSVJknJzc6UnnnhCiomJkdauXev0eQ8fPiwlJSXZJac1Mt+8eVNKTU2VnnjiCWny5MnSpk2brJKHt7kgIiLB6Q8fERGR/bAUiIhIYCkQEZHAUiAiIoGlQEREAkuBOi25XI74+HhMmTIF8+bNQ21tLQBApVI1urtnQUFBo3ELFy5EeXl5q+s3GAxISUnBiBEjLL6gr/6kwA0bNohpvV6Pp556SlwXQmQLLAXqtO677z7s2LEDeXl56NGjB/7+97+L+WvXrkV+fj6ysrKQnp4uCuPs2bMwGo1mXTz26quvIjg4GO+++y4WLVrU6MrV1uzcuRNZWVnQ6XTYtGkTdu7cCU9PTygUintKisiaWApEAMLCwsSNyPr374+f//znAO7cYsDPzw/Xrl0DAOzatQvR0dEAgOzsbKxZs0as45///CfS09MB3Lm/Urdu3bBs2TJERERg9erVWLx4MW7cuHHPtn/88UfMnz8fcXFxiIuLQ2lpKeLj4xEUFITNmzejT58+iI+PBwCMGzcOu3btstnPgYilQJ2e0WhEcXHxPbcXAO7cdKyurk7cg6q0tBSDBw8GAEyePBmFhYWoq6sDAOTk5GDatGkAgAULFuDFF18U6wkPD8fWrVvRrVu3e7axevVqDB8+HDt37sT27dsxYMAA7Nq1CxUVFZgzZw6+//57UQQDBgywyv1tiJrDu6RSp6XVahEfHw+1Wo2QkBCMGjWq0fcrKyuxdOlSrF27Fl263Pn76YcffoCfnx8A4P7778fIkSNx8OBBBAcHo66uDgMHDrQ4x+HDh/Hmm28CANzc3NCtWzdMmTIFMpkMGzZswLPPPis+Y3Bzc4OHhwdu3rzp1HdJJdfFPQXqtOo/UygsLIQkSeIzBQC4efMm5s6di0WLFiEsLEzM9/LyavRYyaSkJOTk5CAnJweJiYlWy1Z/k7bU1NRG08CdxzM6863JybWxFKjT8/b2xssvv4wPP/wQBoMBer0e8+fPR3x8PCZNmtRo2ZCQEFy6dElMh4aGoqKiAnl5eZgyZYrZ2/z444/F7boVCgW2bt0K4M6hrKY+d6hXXV2Nnj17wsPDw5KXSGQ2lgIRgIcffhgDBw5EXl4edu/ejaNHj2L79u3iQe71D3EfO3asuF1xvcmTJ2Po0KHo0aOH2ds7f/68eErWihUrUFJSgtjYWCQmJuLcuXPNjispKcHjjz9u8esjMhfvkkpkAa1Wi2eeeQb/+Mc/xDOH586di5kzZ1r0VLG5c+diw4YNFj9qdcGCBViyZAn69+9v0Tgic7EUiCx06NAhhISEoGvXrkhKSsLAgQOxfv16m29Xr9ejoKAACQkJNt8WdV4sBSIiEviZAhERCSwFIiISWApERCSwFIiISGApEBGR8P8ApmkCFpJRYycAAAAASUVORK5CYII=\n" }, "metadata": {}, "output_type": "display_data" @@ -193,7 +207,10 @@ "i.e. we have no evidence against the null ($p=0.76$)." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { @@ -202,7 +219,7 @@ "outputs": [ { "data": { - "text/plain": "", + "text/plain": "", "image/svg+xml": "\n\n\n\n\n\n%3\n\n\n\nc\n\nc\n\n\n\ny\n\ny\n\n\n\nc--y\n\n0.187\n\n\n\nyhat\n\nŷ\n\n\n\nc--yhat\n\n0.032 (p=0.76)\n\n\n\ny--yhat\n\n0.211\n\n\n\n" }, "execution_count": 11, @@ -226,7 +243,10 @@ "The graph shows the unconditional $R^2$ values across the target $y$, confounder $c$ and predictions $\\hat{y}$." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { @@ -237,26 +257,29 @@ "No let's apply the partial confounder test for H1, that is for a confounded model." ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Permuting: 100%|██████████| 1000/1000 [00:01<00:00, 595.58it/s]\n" + "Permuting: 100%|██████████| 1000/1000 [00:01<00:00, 524.01it/s]\n" ] }, { "data": { - "text/plain": " p ci lower ci upper R2(y,c) R2(y^,c) R2(y,y^)\n0 0.027 0.017867 0.039042 0.187028 0.067903 0.237854", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
pci lowerci upperR2(y,c)R2(y^,c)R2(y,y^)
00.0270.0178670.0390420.1870280.0679030.237854
\n
" + "text/plain": " p ci lower ci upper R2(y,c) R2(y,y^) Expected R2(y^,c) \\\n0 0.027 0.017867 0.039042 0.187028 0.237854 [0.027, 0.044, 0.065] \n\n R2(y^,c) \n0 0.067903 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
pci lowerci upperR2(y,c)R2(y,y^)Expected R2(y^,c)R2(y^,c)
00.0270.0178670.0390420.1870280.237854[0.027, 0.044, 0.065]0.067903
\n
" }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -274,8 +297,9 @@ " 'ci lower' : [ret.p_ci[0]],\n", " 'ci upper' : [ret.p_ci[1]],\n", " 'R2(y,c)' : [ret.r2_y_c],\n", - " 'R2(y^,c)' : [ret.r2_yhat_c],\n", " 'R2(y,y^)' : [ret.r2_y_yhat],\n", + " 'Expected R2(y^,c)': [np.round(ret.expected_r2_yhat_c, 3)],\n", + " 'R2(y^,c)' : [ret.r2_yhat_c]\n", "})" ], "metadata": { @@ -339,7 +363,10 @@ "machine learning models, in prep, 2021.*" ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } } } ], diff --git a/pyproject.toml b/pyproject.toml index d6c8114..5eef7e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mlconfound" -version = "0.20.1" +version = "0.21.0" description = "Tools for analyzing and quantifying effects of counfounder variables on machine learning model predictions." authors = ["Tamas Spisak "] license = "GPL-3.0-or-later"