feat: add ci & wp from jsonpog

- utils: read wps from json files - ci: add camapgin - ci: add automation ci check
Ming-Yan · Oct 16, 2024 · 34de14d · 34de14d
1 parent 1130c66
commit 34de14d
Show file tree

Hide file tree

Showing 16 changed files with 272 additions and 162 deletions.
diff --git a/.github/workflows/BTA_workflow.yml b/.github/workflows/BTA_workflow.yml
@@ -90,12 +90,12 @@ jobs:
         pip install -e .
     - name: BTA addAllTracks workflow test(JP)
       run: |
-         python runner.py --wf BTA_addAllTracks --json metadata/test_bta_run3.json --executor iterative --overwrite 
+         python runner.py --wf BTA_addAllTracks --json metadata/test_bta_run3.json --executor iterative --overwrite --campaign Summer23 --year 2023
     - name: BTA addPFMuons workflow test(BTA for SF)
       run: |
-         python runner.py --wf BTA_addPFMuons --json metadata/test_bta_run3.json --executor iterative --overwrite 
+         python runner.py --wf BTA_addPFMuons --json metadata/test_bta_run3.json --executor iterative --overwrite --campaign Summer23 --year 2023
     
     - name: BTA_ttbar workflow test
       run: |
-         python runner.py --wf BTA_ttbar --json metadata/test_bta_run3.json  --executor iterative --overwrite
+         python runner.py --wf BTA_ttbar --json metadata/test_bta_run3.json  --executor iterative --overwrite --campaign Summer23 --year 2023
 
diff --git a/.github/workflows/QCD_workflow.yml b/.github/workflows/QCD_workflow.yml
@@ -91,7 +91,7 @@ jobs:
         pip install -e .
     
 
-    - name: QCD workflows with correctionlib
+    - name: QCD workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -104,9 +104,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow QCD_sf --json metadata/test_bta_run3.json  --executor iterative  $opts
+        python runner.py --workflow QCD_sf --json metadata/test_bta_run3.json  --executor iterative  --campaign Summer23 --year 2023 $opts
     
-    # - name: QCD mu workflows with correctionlib
+    # - name: QCD mu workflows 
     #   run: |
     #     message=$(git log -1 --pretty=format:'%s')
     #     if [[ $string == *"ci:skip array"* ]]; then
@@ -119,4 +119,4 @@ jobs:
     #     elif [[ $string == *"ci:weight_only"* ]]; then
     #         opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
     #     fi
-    #     python runner.py --workflow QCD_mu --json metadata/test_bta_run3.json --executor iterative  --overwrite $opts
+    #     python runner.py --workflow QCD_mu --json metadata/test_bta_run3.json --executor iterative  --overwrite --campaign Summer23 --year 2023 $opts
diff --git a/.github/workflows/auto.yml b/.github/workflows/auto.yml
@@ -0,0 +1,108 @@
+name: Automation test
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'scripts/plotdataMC.py'
+    - 'scripts/suball.py'
+    - 'scripts/dump_processed.py'
+    - 'scripts/fetch.py'
+    - 'src/BTVNanoCommissioning/utils/sample.py'
+    - '.github/workflows/auto.yml'
+  pull_request_target:
+    branches: [ master ]
+    paths:
+    - 'scripts/plotdataMC.py'
+    - 'scripts/suball.py'
+    - 'scripts/dump_processed.py'
+    - 'scripts/fetch.py'
+    - 'src/BTVNanoCommissioning/utils/sample.py'
+    - '.github/workflows/auto.yml'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.10"]
+
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cvmfs-contrib/github-action-cvmfs@v4
+      with:
+        cvmfs_repositories: 'grid.cern.ch'
+    - name: Set conda environment
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        miniforge-variant: Mambaforge
+        channels: conda-forge,defaults
+        channel-priority: true
+        activate-environment: btv_coffea
+        environment-file: test_env.yml
+        auto-activate-base: false
+
+    - name: Verify environment
+      run: |
+        conda info
+        conda env list
+        conda list
+        
+    - name: Set up proxy
+      # https://awesome-workshop.github.io/gitlab-cms/03-vomsproxy/index.html
+      # continue-on-error: true
+      env:
+        # To genereate secrets use (strip all \n)
+        # base64 -i ~/.globus/usercert.pem | awk NF=NF RS= OFS=
+        # base64 -i ~/.globus/userkey.pem | awk NF=NF RS= OFS=
+        # Cross check roundtrip by adding ``| base64 -d `` and see if same as input
+        GRID_USERKEY: ${{ secrets.GRID_USERKEY }}
+        GRID_USERCERT: ${{ secrets.GRID_USERCERT }}
+        # Read automatically by voms-proxy-init
+        X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir/
+        X509_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses/
+        X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates/
+        X509_DEFAULT_USER_CERT: $HOME/.globus/usercert.pem
+        X509_DEFAULT_USER_KEY: $HOME/.globus/userkey.pem
+      run: |
+        mkdir $HOME/.globus
+        printf $GRID_USERKEY | base64 -d > $HOME/.globus/userkey.pem
+        printf $GRID_USERCERT | base64 -d > $HOME/.globus/usercert.pem
+        # DEBUG: dump decoded cert, cert is public, but don't dump key!
+        # base64 -i $HOME/.globus/usercert.pem
+        chmod 400 $HOME/.globus/userkey.pem
+        openssl rand -out $HOME/.rnd  -hex 256
+        printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms  --debug --pwstdin -certdir $X509_CERT_DIR  -vomses $X509_VOMSES
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
+        
+   
+    - name: Install Repo
+      run: |
+        pip install -e .
+    
+
+    - name: check autmoation scripts
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python scripts/suball.py --scheme default_comissioning --campaign Summer23  --DAS_campaign "*Run2023D*Sep2023*,*Run3Summer23BPixNanoAODv12-130X*" --year 2023 --executor iterative $opts
+    
+
+
diff --git a/.github/workflows/ctag_DY_workflow.yml b/.github/workflows/ctag_DY_workflow.yml
@@ -89,7 +89,7 @@ jobs:
         pip install -e .
     
 
-    - name: ctag muon DY workflows with correctionlib
+    - name: ctag muon DY workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -102,10 +102,10 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative  $opts
+        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023  $opts
     
 
-    - name: ctag electron DY workflows with correctionlib
+    - name: ctag electron DY workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -118,4 +118,4 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts 
diff --git a/.github/workflows/ctag_Wc_workflow.yml b/.github/workflows/ctag_Wc_workflow.yml
@@ -91,7 +91,7 @@ jobs:
         pip install -e .
     
 
-    - name: ctag muon W+c workflows with correctionlib
+    - name: ctag muon W+c workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -104,9 +104,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json  --executor iterative  $opts
+        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json  --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag electron W+c workflows with correctionlib
+    - name: ctag electron W+c workflows
       run: |
         message=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -119,4 +119,4 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --executor iterative  --overwrite $opts
+        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --executor iterative  --overwrite --campaign Summer23 --year 2023 $opts
diff --git a/.github/workflows/ctag_ttbar_workflow.yml b/.github/workflows/ctag_ttbar_workflow.yml
@@ -90,7 +90,7 @@ jobs:
         pip install -e .
 
     
-    - name: ctag semileptonic muon ttbar workflows with correctionlib
+    - name: ctag semileptonic muon ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -103,9 +103,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ctag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag semileptonic electron ttbar workflows with correctionlib
+    - name: ctag semileptonic electron ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -118,9 +118,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi 
-        python runner.py --workflow ectag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ectag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag dileptonic muon ttbar workflows with correctionlib
+    - name: ctag dileptonic muon ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -133,8 +133,8 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi 
-        python runner.py --workflow ctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
-    - name: ctag dileptonic electron ttbar workflows with correctionlib 
+        python runner.py --workflow ctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
+    - name: ctag dileptonic electron ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -147,8 +147,8 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
-    - name: ctag dileptonic emu ttbar workflows with correctionlib
+        python runner.py --workflow ectag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
+    - name: ctag dileptonic emu ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -161,5 +161,5 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow emctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow emctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
 
diff --git a/.github/workflows/ttbar_workflow.yml b/.github/workflows/ttbar_workflow.yml
@@ -88,7 +88,7 @@ jobs:
       run: |
         pip install -e .
 
-    - name:  btag semileptonic ttbar workflows with correctionlib
+    - name:  btag semileptonic ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -101,10 +101,10 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
     
 
-    - name: btag dileptonic ttbar workflows with correctionlib
+    - name: btag dileptonic ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -117,5 +117,5 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
 
diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml
@@ -89,9 +89,9 @@ jobs:
         pip install -e .
 
    
-    - name: btag dileptonic ttbar workflows with correctionlib
+    - name: btag dileptonic ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         
-        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --year 2023 --campaign Summer23 $opts
 
diff --git a/runner.py b/runner.py
@@ -46,7 +46,11 @@ def retry_handler(exception, task_record):
 
 ## From condor/submitter.py https://github.com/cms-btv-pog/BTVNanoCommissioning/blob/9edb9ed6bb0b28730b8de9e5aa1142ec4fdf74b7/condor/submitter.py
 # create t
+
+
 def make_tarfile(output_filename, source_dir, exclude_dirs=[]):
+    import tarfile
+
     with tarfile.open(output_filename, "w:gz") as tar:
         for root, dirs, files in os.walk(source_dir):
             dirs[:] = [
@@ -524,6 +528,7 @@ def debug_parser(parser):
                             "BTVNanoCommissioning.egg-info",
                         ],
                     )
+            import shutil
 
             # Create job dir
             job_dir = f"jobs_{args.jobName}"

diff --git a/src/BTVNanoCommissioning/utils/array_writer.py b/src/BTVNanoCommissioning/utils/array_writer.py
@@ -67,7 +67,7 @@ def array_writer(
             for obj in kinOnly:
                 if "MET" in obj and ("pt" != kin or "phi" != kin):
                     continue
-                if (obj != "Muon" and obj != "SoftMuon") and (
+                if (obj != "SelMuon" and obj != "SoftMuon") and (
                     "pfRelIso04_all" == kin or "d" in kin
                 ):
                     continue

diff --git a/src/BTVNanoCommissioning/utils/correction.py b/src/BTVNanoCommissioning/utils/correction.py
@@ -262,7 +262,7 @@ def load_SF(year, campaign, syst=False):
         elif SF == "JME":
             # year = int(re.search(r"\d+", campaign).group())
             if type(config[campaign]["JME"]) == str:
-                correct_map["JME"] = load_jmefactory(campaign)
+                correct_map["JME"] = load_jmefactory(year, campaign)
             elif os.path.exists(
                 f"/cvmfs/cms.cern.ch/rsync/cms-nanoAOD/jsonpog-integration/POG/JME/{year}_{campaign}/jet_jerc.json.gz"
             ):
@@ -366,8 +366,8 @@ def jetveto(jets, correct_map):
 
 ##JEC
 # FIXME: would be nicer if we can move to correctionlib in the future together with factory and workable
-def load_jmefactory(campaign):
-    _jet_path = f"BTVNanoCommissioning.data.JME.{campaign}"
+def load_jmefactory(year, campaign):
+    _jet_path = f"BTVNanoCommissioning.data.JME.{year}_{campaign}"
     with importlib.resources.path(_jet_path, config[campaign]["JME"]) as filename:
         with gzip.open(filename) as fin:
             jmestuff = cloudpickle.load(fin)