lepton veto templates + speed up

src/HHbbVV/postprocessing/bash_scripts/NonresTemplates.sh

@@ -40,12 +40,11 @@ if [[ -z $TAG ]]; then
   exit 1
 fi
 
-# for year in 2016APV 2016 2017 2018
-for year in 2016 2017 2018
+for year in 2016APV 2016 2017 2018
 do
     python -u postprocessing.py --year $year --data-dir "$data_dir" --templates \
     --bdt-preds-dir "$MAIN_DIR/../data/skimmer/24Mar6AllYearsBDTVars/24_03_07_new_samples_max_depth_5/inferences" \
-    --control-plots --control-plot-vars "BDTScore" \
     --plot-dir "${MAIN_DIR}/plots/PostProcessing/$TAG" \
     --template-dir "templates/$TAG" --plot-shifts
+    # --control-plots --control-plot-vars "BDTScore" \
 done

src/HHbbVV/postprocessing/corrections.py

@@ -64,6 +64,7 @@ def apply_txbb_sfs(
     bb_mask: pd.DataFrame,
     year: str,
     weight_key: str = "finalWeight",
+    do_shifts: bool = True,
 ):
     """Applies nominal values to ``weight_key`` and stores up down variations"""
     if year not in txbb_sf_lookups:
@@ -72,17 +73,19 @@ def apply_txbb_sfs(
     bb_txbb = utils.get_feat(events, "bbFatJetParticleNetMD_Txbb", bb_mask)
     bb_pt = utils.get_feat(events, "bbFatJetPt", bb_mask)
 
-    for var in ["up", "down"]:
-        if len(events[weight_key]):
-            events[f"{weight_key}_txbb_{var}"] = events[weight_key] * txbb_sf_lookups[year][var](
-                bb_txbb, bb_pt
-            )
-        else:
-            events[f"{weight_key}_txbb_{var}"] = events[weight_key]
+    if do_shifts:
+        for var in ["up", "down"]:
+            if len(events[weight_key]):
+                events[f"{weight_key}_txbb_{var}"] = events[weight_key] * txbb_sf_lookups[year][
+                    var
+                ](bb_txbb, bb_pt)
+            else:
+                events[f"{weight_key}_txbb_{var}"] = events[weight_key]
 
     if len(events[weight_key]):
         txbb_nom = txbb_sf_lookups[year]["nom"](bb_txbb, bb_pt)
-        for wkey in utils.get_all_weights(events):
+        scale_wkeys = utils.get_all_weights(events) if do_shifts else [weight_key]
+        for wkey in scale_wkeys:
             if len(events[wkey].shape) > 1:
                 events[wkey] *= txbb_nom[:, np.newaxis]
             else:

src/HHbbVV/postprocessing/postprocessing.py

@@ -1640,6 +1640,10 @@ def get_templates(
         Dict[str, Hist]: dictionary of templates, saved as hist.Hist objects.
 
     """
+    import time
+
+    start = time.time()
+
     if weight_shifts is None:
         weight_shifts = {}
     do_jshift = jshift != ""
@@ -1652,6 +1656,8 @@ def get_templates(
         if rname == "lpsf":
             continue
 
+        print(f"{rname} Region: {time.time() - start:.2f}")
+
         if not do_jshift:
             print(rname)
 
@@ -1664,6 +1670,7 @@ def get_templates(
             jshift=jshift,
             weight_key=weight_key,
         )
+        print(f"Selection: {time.time() - start:.2f}")
 
         if template_dir != "":
             cf.to_csv(f"{template_dir}/cutflows/{year}/{rname}_cutflow{jlabel}.csv")
@@ -1683,12 +1690,21 @@ def get_templates(
             sig_bb_mask = bb_masks[sig_key][sel[sig_key]]
 
             if pass_region:
-                # scale all signal weights by LP SF
+                # scale all signal weights by LP SF (if not doing a j shift)
                 if lpsfs:
-                    for wkey in utils.get_all_weights(sig_events[sig_key]):
+                    scale_wkeys = (
+                        utils.get_all_weights(sig_events[sig_key])
+                        if not do_jshift
+                        else [weight_key]
+                    )
+                    for wkey in scale_wkeys:
                         sig_events[sig_key][wkey] *= systematics[sig_key]["lp_sf"]
 
-                corrections.apply_txbb_sfs(sig_events[sig_key], sig_bb_mask, year, weight_key)
+                corrections.apply_txbb_sfs(
+                    sig_events[sig_key], sig_bb_mask, year, weight_key, do_shifts=not do_jshift
+                )
+
+        print(f"Tagger SFs: {time.time() - start:.2f}")
 
         # if not do_jshift:
         #     print("\nCutflow:\n", cf)
@@ -1768,6 +1784,8 @@ def get_templates(
                                 utils.get_key_index(h, f"{sample}_{wshift}_down"), :
                             ] = shape_down
 
+        print(f"Histograms: {time.time() - start:.2f}")
+
         if pass_region:
             # blind signal mass windows in pass region in data
             for i, shape_var in enumerate(shape_vars):
@@ -1797,6 +1815,7 @@ def get_templates(
         ################################
 
         if plot_dir != "" and (not do_jshift or plot_shifts):
+            print(f"Plotting templates: {time.time() - start:.2f}")
             if plot_sig_keys is None:
                 plot_sig_keys = sig_keys
 
@@ -2080,7 +2099,7 @@ def parse_args():
     parser.add_argument(
         "--lepton-veto",
         help="lepton vetoes: None, Hbb, or HH",
-        default=["None"],
+        default=["Hbb"],
         nargs="*",
         type=str,
     )

src/HHbbVV/postprocessing/templates/24Mar8NonresHbbLeptonVeto/cutflows/2016/fail_cutflow.csv

@@ -0,0 +1,25 @@
+,Pre-selection,QCD SF,LP SF,bbFatJetPt >= 300,VVFatJetPt >= 300,0.8 ≤ bbFatJetParticleNetMD_Txbb < 0.9735,0 ≤ nGoodElectronsHbb < 0.9,0 ≤ nGoodMuonsHbb < 0.9
+HHbbVV,0.9147081031687907,0.9147081031687907,0.9147081031687907,0.9088022319312213,0.9037950499185726,0.30175414428376585,0.30112232829530405,0.2990202748826515
+ggHH_kl_2p45_kt_1_HHbbVV,0.6688447704532685,0.6688447704532685,0.6688447704532685,0.6649531572120448,0.6617354385600798,0.2299036391879109,0.2295289421882006,0.2280924478485079
+ggHH_kl_5_kt_1_HHbbVV,0.6956895087373941,0.6956895087373941,0.6956895087373941,0.6920514447111688,0.6892047021562906,0.3063302486063932,0.30550315963569236,0.30391351000104144
+ggHH_kl_0_kt_1_HHbbVV,1.1944766869755938,1.1944766869755938,1.1944766869755938,1.1863493648635597,1.17930333767202,0.39586526060371247,0.39514188691284224,0.39256694301449424
+VBFHHbbVV,0.017115373482989717,0.017115373482989717,0.017115373482989717,0.017030078444247275,0.016924919012766975,0.00579761879633008,0.0057790354556334165,0.0057248101820241174
+qqHH_CV_1_C2V_0_kl_1_HHbbVV,9.060969753010587,9.060969753010587,9.060969753010587,9.041934826728792,9.02728294983494,2.4426924669829826,2.43781489891332,2.4243925435159057
+qqHH_CV_1p5_C2V_1_kl_1_HHbbVV,15.269096111629358,15.269096111629358,15.269096111629358,15.23647158421826,15.20829496277662,4.13073889572393,4.120987792038875,4.10015685148764
+qqHH_CV_1_C2V_1_kl_2_HHbbVV,0.0173959132140592,0.0173959132140592,0.0173959132140592,0.017299990201845,0.017214419089199705,0.00846784274434387,0.008440107570873445,0.008373264656125586
+qqHH_CV_1_C2V_2_kl_1_HHbbVV,8.138882127595444,8.138882127595444,8.138882127595444,8.123943352838433,8.11286122859791,2.1892512907726895,2.1837678445588895,2.1717354106435
+qqHH_CV_1_C2V_1_kl_0_HHbbVV,0.04617824244896124,0.04617824244896124,0.04617824244896124,0.0459311591300662,0.04565497610400018,0.014190872302543785,0.01416197212352274,0.014112105993637325
+QCD,717986.216814939,970717.8146709627,970717.8146709627,962216.3978416026,955328.3949004482,765540.0455045162,763948.8156746186,761029.7753251398
+TT,63126.80979180276,63126.80979180276,63126.80979180276,62714.21430804499,62290.703474200425,49529.77449791308,44329.76646173106,39159.45860664165
+ST,5417.824558008516,5417.824558008516,5417.824558008516,5380.176293677084,5350.773696634727,4335.103148326725,3970.1436871856044,3563.758325231233
+W+Jets,9487.889178263631,9487.889178263631,9487.889178263631,9424.292575831972,9375.118673797197,7834.787103754956,7814.4993526839335,7789.446400547657
+Z+Jets,12055.822300436781,12055.822300436781,12055.822300436781,11968.016139487008,11904.514452147456,6790.915177024769,6773.707406230397,6734.8455822324595
+Diboson,544.751536187187,544.751536187187,544.751536187187,541.5192000426408,539.5255957808083,336.0954273567496,320.18549221259616,303.48286314956806
+ggFHbb,120.74204174793252,120.74204174793252,120.74204174793252,120.02707196890958,119.27175261578469,38.999571077551494,38.97813391667,38.788894151509346
+VBFHbb,32.24173303425971,32.24173303425971,32.24173303425971,32.089487243194796,31.921442997500897,9.845327900626657,9.826780907273822,9.728638615140152
+ZHbb,32.70020301080715,32.70020301080715,32.70020301080715,32.51666448179822,32.40029638484181,9.92855058278212,9.896574590344532,9.807368124790582
+WHbb,56.014320595270185,56.014320595270185,56.014320595270185,55.69337399997595,55.51504919284625,19.11120104088777,19.06683717160466,18.91991542799836
+ggZHbb,4.8805631912185055,4.8805631912185055,4.8805631912185055,4.852821845932874,4.827078996358681,1.9281741450543308,1.9243343671398156,1.9080303961975615
+ttHbb,156.94119216557863,156.94119216557863,156.94119216557863,155.9939482895008,154.97672213467678,97.04622661247781,85.884977667534,74.2696929751333
+HWW,80.56791059341468,80.56791059341468,80.56791059341468,80.1332628058381,79.59897023669294,63.60672027477934,52.77270242090221,43.58177362660231
+Data,1061835.0,1061835.0,1061835.0,1061835.0,1061835.0,875363.0,866315.0,855954.0

src/HHbbVV/postprocessing/templates/24Mar8NonresHbbLeptonVeto/cutflows/2016/fail_cutflow_JER_down.csv

@@ -0,0 +1,25 @@
+,Pre-selection,QCD SF,LP SF,bbFatJetPt_JER_down >= 300,VVFatJetPt_JER_down >= 300,0.8 ≤ bbFatJetParticleNetMD_Txbb < 0.9735,0 ≤ nGoodElectronsHbb < 0.9,0 ≤ nGoodMuonsHbb < 0.9,bbFatJetPt >= 300,VVFatJetPt >= 300
+HHbbVV,0.9147081031687907,0.9147081031687907,0.9147081031687907,0.9086202562710456,0.9032729107620694,0.30136641526403185,0.30073459927557006,0.29863254586291754,,
+ggHH_kl_2p45_kt_1_HHbbVV,0.6688447704532685,0.6688447704532685,0.6688447704532685,0.6650215844188102,0.6616978818114161,0.2299215770147511,0.2295468800150408,0.22811058948062643,,
+ggHH_kl_5_kt_1_HHbbVV,0.6956895087373941,0.6956895087373941,0.6956895087373941,0.6920552540546806,0.6889646563286256,0.3060371693804962,0.30521008040979536,0.30362043077514445,,
+ggHH_kl_0_kt_1_HHbbVV,1.1944766869755938,1.1944766869755938,1.1944766869755938,1.1860618192897887,1.1784620440607205,0.395068883543628,0.39434550985275774,0.3918356130348024,,
+VBFHHbbVV,0.017115373482989717,0.017115373482989717,0.017115373482989717,0.017030649078712407,0.016911806406707055,0.005799982669244992,0.005778877997696238,0.00572465272408694,,
+qqHH_CV_1_C2V_0_kl_1_HHbbVV,9.060969753010587,9.060969753010587,9.060969753010587,9.043185519276458,9.02815771027725,2.442574423607405,2.437696855537743,2.424358564304257,,
+qqHH_CV_1p5_C2V_1_kl_1_HHbbVV,15.269096111629358,15.269096111629358,15.269096111629358,15.23819151213184,15.21085147798668,4.130793913429084,4.12104280974403,4.100149537665231,,
+qqHH_CV_1_C2V_1_kl_2_HHbbVV,0.0173959132140592,0.0173959132140592,0.0173959132140592,0.01731116970053962,0.01721529909865581,0.008463374900600396,0.008435639727129969,0.008371134586267433,,
+qqHH_CV_1_C2V_2_kl_1_HHbbVV,8.138882127595444,8.138882127595444,8.138882127595444,8.124193038879898,8.112651573642607,2.1889068742669315,2.183430839763076,2.171398405847687,,
+qqHH_CV_1_C2V_1_kl_0_HHbbVV,0.04617824244896124,0.04617824244896124,0.04617824244896124,0.04594810143671735,0.04559847624848856,0.014189296739562046,0.014160396560541001,0.014110530430655587,,
+QCD,717986.216814939,970717.8146709627,970717.8146709627,962173.4045094895,955322.8744764234,765482.4698877732,763900.467137054,760999.9349758499,,
+TT,63126.80979180276,63126.80979180276,63126.80979180276,62673.79614323329,62216.48155564703,49469.34874682441,44281.27331024803,39123.833732942854,,
+ST,5417.824558008516,5417.824558008516,5417.824558008516,5377.353115354821,5346.142315388623,4331.322795140036,3966.3474558289413,3560.351662865544,,
+W+Jets,9487.889178263631,9487.889178263631,9487.889178263631,9420.543436555281,9370.593747378542,7830.22486542637,7809.937114355349,7785.400628667634,,
+Z+Jets,12055.822300436781,12055.822300436781,12055.822300436781,11963.681352389223,11895.51750751837,6787.97110352534,6771.117645697598,6732.151861603023,,
+Diboson,544.751536187187,544.751536187187,544.751536187187,541.5333279703598,539.5308044193844,336.208740931941,320.29880578778756,303.6720365276853,,
+ggFHbb,120.74204174793252,120.74204174793252,120.74204174793252,120.07126024952862,119.40043805803371,39.08114940009345,39.05971223921196,38.87047247405131,,
+VBFHbb,32.24173303425971,32.24173303425971,32.24173303425971,32.09098572075379,31.915490654706662,9.839370046175077,9.820823052822242,9.722680760688572,,
+ZHbb,32.70020301080715,32.70020301080715,32.70020301080715,32.51229709989521,32.39048595448399,9.917885138845206,9.885909146407617,9.796702680853667,,
+WHbb,56.014320595270185,56.014320595270185,56.014320595270185,55.68961912767411,55.51113359847001,19.10746601933774,19.063102150054632,18.916180406448333,,
+ggZHbb,4.8805631912185055,4.8805631912185055,4.8805631912185055,4.850277343583236,4.823413122979026,1.925879544563995,1.9220397666494795,1.905669229852938,,
+ttHbb,156.94119216557863,156.94119216557863,156.94119216557863,155.94705721730352,154.8512284574327,96.96277123860979,85.81557467540266,74.21606240134093,,
+HWW,80.56791059341468,80.56791059341468,80.56791059341468,80.06917628289587,79.44977797155293,63.490065562691505,52.68396852595001,43.51687144284523,,
+Data,1061835.0,1061835.0,1061835.0,,,875363.0,866315.0,855954.0,1061835.0,1061835.0

src/HHbbVV/postprocessing/templates/24Mar8NonresHbbLeptonVeto/cutflows/2016/fail_cutflow_JER_up.csv

@@ -0,0 +1,25 @@
+,Pre-selection,QCD SF,LP SF,bbFatJetPt_JER_up >= 300,VVFatJetPt_JER_up >= 300,0.8 ≤ bbFatJetParticleNetMD_Txbb < 0.9735,0 ≤ nGoodElectronsHbb < 0.9,0 ≤ nGoodMuonsHbb < 0.9,bbFatJetPt >= 300,VVFatJetPt >= 300
+HHbbVV,0.9147081031687907,0.9147081031687907,0.9147081031687907,0.9087910422523933,0.9039911771807736,0.30181419690901495,0.30118238092055316,0.29909776577343905,,
+ggHH_kl_2p45_kt_1_HHbbVV,0.6688447704532685,0.6688447704532685,0.6688447704532685,0.6648232888972543,0.6615304688795937,0.22983874093189527,0.229464043932185,0.22802458789429447,,
+ggHH_kl_5_kt_1_HHbbVV,0.6956895087373941,0.6956895087373941,0.6956895087373941,0.6919837261137559,0.6895359500254084,0.30660509644133715,0.3057780074706363,0.30418835783598547,,
+ggHH_kl_0_kt_1_HHbbVV,1.1944766869755938,1.1944766869755938,1.1944766869755938,1.1863320607318555,1.1793065385470574,0.39609963331825193,0.39537625962738165,0.3927919024361102,,
+VBFHHbbVV,0.017115373482989717,0.017115373482989717,0.017115373482989717,0.017017363606822954,0.016929960784848687,0.005800912409835287,0.005782329069138623,0.005728103795529325,,
+qqHH_CV_1_C2V_0_kl_1_HHbbVV,9.060969753010587,9.060969753010587,9.060969753010587,9.040636795335303,9.026372449520458,2.442600216890031,2.4377226488203685,2.424321715421426,,
+qqHH_CV_1p5_C2V_1_kl_1_HHbbVV,15.269096111629358,15.269096111629358,15.269096111629358,15.235710988413302,15.208705941882577,4.131227301165773,4.121476197480719,4.100645256929484,,
+qqHH_CV_1_C2V_1_kl_2_HHbbVV,0.0173959132140592,0.0173959132140592,0.0173959132140592,0.017297188158115442,0.01721631337755096,0.00847691805438527,0.008449182880914844,0.008380206286157436,,
+qqHH_CV_1_C2V_2_kl_1_HHbbVV,8.138882127595444,8.138882127595444,8.138882127595444,8.123037690630078,8.112049494318502,2.188880801375314,2.1833973551615142,2.171395817825956,,
+qqHH_CV_1_C2V_1_kl_0_HHbbVV,0.04617824244896124,0.04617824244896124,0.04617824244896124,0.045925054049681675,0.04568813387406423,0.014201551410960465,0.01417265123193942,0.014122785102054006,,
+QCD,717986.216814939,970717.8146709627,970717.8146709627,962458.1587531276,955401.3465800203,765705.1233537928,764101.3332905469,761176.544549736,,
+TT,63126.80979180276,63126.80979180276,63126.80979180276,62748.65359108715,62350.558550020374,49581.08106018262,44369.23332839295,39189.123306614805,,
+ST,5417.824558008516,5417.824558008516,5417.824558008516,5380.019529035428,5352.781145230527,4336.103656997989,3970.877734250647,3563.181072832113,,
+W+Jets,9487.889178263631,9487.889178263631,9487.889178263631,9424.098448265346,9376.427000646685,7837.1106797885595,7816.822928717539,7791.573471785453,,
+Z+Jets,12055.822300436781,12055.822300436781,12055.822300436781,11970.280347915328,11907.501953889905,6792.360403864935,6775.275920925513,6736.3344008445665,,
+Diboson,544.751536187187,544.751536187187,544.751536187187,541.0881926888653,539.2414054191454,335.99317001586985,320.005274959971,303.302645896943,,
+ggFHbb,120.74204174793252,120.74204174793252,120.74204174793252,120.07042524743345,119.27825223478067,38.98833072689881,38.96689356601732,38.77765380085667,,
+VBFHbb,32.24173303425971,32.24173303425971,32.24173303425971,32.11233185743809,31.972834248054944,9.851790818054113,9.83324382470128,9.73510153256761,,
+ZHbb,32.70020301080715,32.70020301080715,32.70020301080715,32.50629199227846,32.37127325754241,9.930739682671923,9.898763690234336,9.809557224680384,,
+WHbb,56.014320595270185,56.014320595270185,56.014320595270185,55.67763486575412,55.48629077310594,19.102692705898072,19.05832883661496,18.911407093008663,,
+ggZHbb,4.8805631912185055,4.8805631912185055,4.8805631912185055,4.8517784616619,4.8285634978383225,1.9299360112620434,1.926057142399399,1.9095143794842424,,
+ttHbb,156.94119216557863,156.94119216557863,156.94119216557863,156.03648620372667,155.10345329641785,97.15099751928513,85.95956128176189,74.32330052260997,,
+HWW,80.56791059341468,80.56791059341468,80.56791059341468,80.16639644289663,79.67713590709836,63.657224321188984,52.8022508024436,43.60289736509357,,
+Data,1061835.0,1061835.0,1061835.0,,,875363.0,866315.0,855954.0,1061835.0,1061835.0