Merge branch 'ANNIEsoft:Application' into Application

Setup.sh

@@ -12,7 +12,7 @@ export CLHEP_DIR=${ToolDAQapp}/ToolDAQ/2.4.0.2/CLHEP_install
 
 export LD_LIBRARY_PATH=`pwd`/lib:${ToolDAQapp}/lib:${ToolDAQapp}/ToolDAQ/zeromq-4.0.7/lib:${ToolDAQapp}/ToolDAQ/boost_1_66_0/install/lib:${ToolDAQapp}/ToolDAQ/root/lib:${ToolDAQapp}/ToolDAQ/WCSimLib/:${ToolDAQapp}/ToolDAQ/MrdTrackLib/src:${ToolDAQapp}/ToolDAQ/RATEventLib/lib/:${ToolDAQapp}/UserTools/PlotWaveforms:${ToolDAQapp}/ToolDAQ/log4cpp/lib:${ToolDAQapp}/ToolDAQ/Pythia6Support/v6_424/lib:${CLHEP_DIR}/lib:${ToolDAQapp}/ToolDAQ/LHAPDF-6.3.0/install/lib:${ToolDAQapp}/ToolDAQ/GENIE-v3-master/lib:${ToolDAQapp}/ToolDAQ/Reweight-3_00_04_ub3/lib:$LD_LIBRARY_PATH
 
-export ROOT_INCLUDE_PATH=${ToolDAQapp}/ToolDAQ/WCSimLib/include/:${ToolDAQapp}/ToolDAQ/MrdTrackLib/include:${ToolDAQapp}/ToolDAQ/RATEventLib/include/:${ToolDAQapp}/UserTools/PlotWaveforms:$ROOT_INCLUDE_PATH
+export ROOT_INCLUDE_PATH=${ToolDAQapp}/ToolDAQ/boost_1_66_0/install/include:${ToolDAQapp}/ToolDAQ/WCSimLib/include/:${ToolDAQapp}/ToolDAQ/MrdTrackLib/include:${ToolDAQapp}/ToolDAQ/RATEventLib/include/:${ToolDAQapp}/UserTools/PlotWaveforms:$ROOT_INCLUDE_PATH
 
 export PYTHIA6_DIR=${ToolDAQapp}/ToolDAQ/Pythia6Support/v6_424/
 export LHAPATH=${ToolDAQapp}/ToolDAQ/LHAPDF-6.3.0/install/share/LHAPDF/

UserTools/CNNImage/CNNImage.h

@@ -3,6 +3,10 @@
 
 #include <string>
 #include <fstream>
+#include <iostream>
+#include <vector>
+
+#include <boost/tokenizer.hpp>
 
 #include "TH2F.h"
 #include "TMath.h"
@@ -12,18 +16,18 @@
 
 
 /**
- * \class CNNImage
- *
- * The tool CNNImage is supposed to create custom csv input files for CNN classification processes. The framework relies on the EventDisplay tool and basically feeds that geometric information of the hit pattern into a matrix format
+* \class CNNImage
+*
+* The tool CNNImage is supposed to create custom csv input files for CNN classification processes.
+*    The framework relies on the EventDisplay tool and basically feeds that geometric information of the hit pattern
+*    into a matrix format
 *
 * $Author: M.Nieslony $
 * $Date: 2019/08/09 10:44:00 $
 * Contact: mnieslon@uni-mainz.de
 */
 
 class CNNImage: public Tool {
-
-
  public:
 
   CNNImage(); ///< constructor for CNNImage class
@@ -35,6 +39,18 @@ class CNNImage: public Tool {
   void ConvertPositionTo2D_Top(Position xyz_pos, double &x, double &y); ///<Convert 3D position into 2D rolled up coordinates (top PMTs only)
   void ConvertPositionTo2D_Bottom(Position xyz_pos, double &x, double &y); ///<Convert 3D position into 2D rolled up coordinates (bottom PMTs only)
 
+  void GetPMTPositions(std::map<std::string, std::map<unsigned long, Detector*> >* Detectors);
+  void OrderPMTPositions();
+
+  void GetLAPPDPositions(std::map<std::string, std::map<unsigned long, Detector*> >* Detectors);
+  void OrderLAPPDPositions();
+
+  void StaticPMTBinning(std::map<unsigned long, std::pair<int, int> >&m);
+  void GeometricPMTBinning(std::map<unsigned long, std::pair<int, int> > &m, TH2F *geometric_hist);
+
+  void ReadoutMRD();
+
+  void PopulateHistInFilePMT(ofstream &outfile, TH2F* hist, int binX, int binY);
 
  private:
 
@@ -43,13 +59,20 @@ class CNNImage: public Tool {
   std::string detector_config;
   std::string data_mode;  //Charge, Time
   std::string save_mode;  //How is the PMT information supposed to be written out? Geometric/PMT-wise
+  bool use_LAPPDs;
+  bool write_to_file;
   int dimensionX;        //dimension of the CNN image in x-direction
   int dimensionY;        //dimension of the CNN image in y-direction
   int dimensionLAPPD;    //dimension of LAPPD CNN images (both directions, square)
   bool includeTopBottom;
-  bool isData; 
+
+  std::map<unsigned long, std::pair<int, int> > static_pmt_mapping;
+  std::map<unsigned long, std::pair<int, int> > geometric_pmt_mapping;
+
+  bool isData;
   int verbosity;
 
+
   // ANNIEEvent variables
   int runnumber;
   int subrunnumber;
@@ -71,6 +94,20 @@ class CNNImage: public Tool {
   // RecoEvent variables
   int nrings;
 
+  // Hists
+  TH2F *hist_cnn;
+  TH2F *hist_cnn_abs;
+  TH2F *hist_cnn_time;
+  TH2F *hist_cnn_time_first;
+  TH2F *hist_cnn_time_abs;
+  TH2F *hist_cnn_time_first_abs;
+  TH2F *hist_cnn_pmtwise;
+  TH2F *hist_cnn_abs_pmtwise;
+  TH2F *hist_cnn_time_pmtwise;
+  TH2F *hist_cnn_time_first_pmtwise;
+  TH2F *hist_cnn_time_abs_pmtwise;
+  TH2F *hist_cnn_time_first_abs_pmtwise;
+
   // Geometry variables
   double tank_radius;
   double tank_height;
@@ -98,6 +135,15 @@ class CNNImage: public Tool {
   double min_time_pmts, max_time_pmts, min_time_lappds, max_time_lappds;
   double min_time_first_pmts, max_time_first_pmts, min_time_first_lappds, max_time_first_lappds;
 
+  // MRD information
+  int num_mrd_paddles_cluster=0;
+  int num_mrd_layers_cluster=0;
+  int num_mrd_conslayers_cluster=0;
+  int num_mrd_adjacent_cluster=0;
+  double mrd_padperlayer_cluster=0.;
+  bool layer_occupied_cluster[11] = {0};
+  double mrd_paddlesize_cluster[11];
+
   // Detectorkey layout organization
   std::map<unsigned long, int> channelkey_to_pmtid;
   std::map<int,unsigned long> pmt_tubeid_to_channelkey;

UserTools/CNNImage/README.md

@@ -1,26 +1,89 @@
 # CNNImage
 
-CNNImage creates ANNIE event display information in a csv-file format that can directly be loaded into Machine Learning classifier frameworks like CNNs for image classification purposes. Currently only PMT information from the side PMTs is loaded, no information from the top/bottom PMTs and LAPPDs is used.
+CNNImage creates ANNIE event display information in matrix format that
+can directly be loaded and used by Machine Learning classifier frameworks like CNNs for image/event 
+classification purposes. Currently only PMT information is loaded, no information from the LAPPDs is used.
+
+The event display created by this tool is given in a rectangular format (static mapping mode: 16x10, 
+geometric mapping mode: user-defined shape), and produces normalized and absolute charge and time 
+(time of first hit, and average hit time) maps that are stored in boost stores or may be written 
+to csv files.
 
 ## Data
 
-CNNImage creates multiple `.csv`-files and one `.root`-file. Each csv-file contains the event display information in single rows  per event for a specific event type (PMTs/LAPPDs, charge/time), whereas the root-file provides the same information in a 2D histogram format.
+CNNImage creates multiple `.csv`-files and one `.root`-file, when `WriteToFile` 
+is set to 1 in the config file. Each csv-file contains the event display information in single rows  
+per event for a specific event type (PMTs/LAPPDs, charge/time), whereas the root-file provides the 
+same information in a 2D histogram format, where each event gets its own histogram.
+
+The PMT data can be written Static mode, with a pre-defined mapping (16x10 Matrix) or Geometric mode 
+(every pixel is defined by its geometric location, if a PMT happens to be within the pixel its value 
+is added to the pixel value, PMTs can overlap. Top and bottom caps are given by circles and not 
+squished into rows). The configuration variables `DimensionX` and `DimensionY` hence only apply when
+using the `Geometric` save mode and not for the `Static` save mode. 
+
+LAPPDs are not yet supported in the static mapping. Further, static mapping does not support 
+dynamically resizing the output matrix size. 
+
+The `DataMode` configuration specifies whether the average hit time per PMT is calculated as a simple
+mean (`Normal`) or weighted by the charge (`Charge`).
 
-The PMT data can be written PMT-wise (every pixel is reserved for just 1 PMT, no spaces in between) or geometrically (every pixel is defined by its geometric location, if a PMT happens to be within the pixel its value is added to the pixel value). The configuration variables `DimensionX` and `DimensionY` hence only apply when using the `Geometric` save mode and not for the `PMT-wise` save mode. The dimensions for the `PMT-wise` save mode are simply given by the number of PMTs in x- and y-direction. When choosing `PMT-wise`, the LAPPD information is also saved per LAPPD in a 20x20cm grid.
+## Boost store
+The following data are provided within the RecoEvent boost store
+```
+m_data->Stores.at("RecoEvent")->Set("CNNImageCharge", cnn_image_charge);
+m_data->Stores.at("RecoEvent")->Set("CNNImageChargeAbs", cnn_image_charge_abs);
+m_data->Stores.at("RecoEvent")->Set("CNNImageTime", cnn_image_time);
+m_data->Stores.at("RecoEvent")->Set("CNNImageTimeFirst", cnn_image_time_first);
+```
+where the images are implemented as
+```
+std::vector<double> cnn_image_charge(cnn_image_size, 0);
+```
 
-The `DataMode` configuration specifies whether the average hit time per PMT is calculated as a simple mean (`Normal`) or weighted by the charge (`Charge-Weighted`).
+## Binning in Static mapping mode
+The Static PMT binning is defined in the `CNNImage::StaticPMTBinning` method. The vertical (y) and 
+horizontal (x/phi) binning is implemented (in detector coordinates) as
+```
+const std::vector<double> vertical_bins = {-180.0, -160.0, -125.0, -75.0, -30.0, 10.0, 30, 75.0, 130.0};
+std::vector<double> phi_bins;
+    for (int i = 0; i < 17; i++) {
+    phi_bins.push_back(i * 2 * TMath::Pi() / 16); //16 bins for 16 PMTs next to eachother
+}
+```
+The bottom and top PMTs are split into two rows, where the inner circle of the tank top PMTs correspond
+to the first row, the inner circle of the tank bottom PMTs correspond to the last row in the produced
+output.
 
+## Prerequisites
+The tool needs the following tools to be loaded at some point beforehand:
+```
+MCParticleProperties
+MCRecoEventLoader
+ClusterFinder
+DigitBuilder
+TimeClustering
+EventSelector
+EnergyExtractor
+```
 
 ## Configuration
 
-CNNImage uses the following configuration variables:
+CNNImage uses the following recommended configuration variables:
 
 ```
-verbosity 1     
-DataMode Normal			#options: Normal / Charge-Weighted
-SaveMode Geometric		#options: Geometric / PMT-wise
-DimensionX 16			#choose something suitable (32/64/...)
-DimensionY 20			#choose something suitable (32/64/...)
-OutputFile beam_test0		#csv file name
-DetectorConf ANNIEp2v6		#specify the detector version used in simulation
+verbosity 0
+DataMode Normal             #options: Charge/Time/Normal
+SaveMode Static             #options: Static / Geometric
+DimensionX 16               #choose something suitable (32/64/...) Relevant only for geometric mode
+DimensionY 10               #choose something suitable (32/64/...)
+OutputFile your_base_name   #base name of output files, if WriteToFile == 1
+
+DimensionLAPPD 5            #Size of the LAPPD pannal 
+IncludeTopBottom 1          #Whether top and bottom should be included
+                            #IncludeTopBottom is only relevant for geometric mode
+DetectorConf ANNIEp2v7
+useLAPPDs 0                 
+WriteToFile 0              
+
 ```

UserTools/RingCounting/README.md

@@ -6,32 +6,51 @@ For this a CNN-based machine learning approach is used.
 This Tool uses PMT data in the CNNImage format (see UserTools/CNNImage). For further details on the tool and
 the models used (including performance etc.) see the documentation found on the anniegpvm-machines at (**Todo**)
 ```
-/pnfs/annie/persistent/users/dschmid/RingCountingStore/documentation/
-```
+/pnfs/annie/persistent/reconstruction/RingCounting/RingCountingStore/documentation/
+``` 
+
 ---
 
 All models can be found at
 ```
-/pnfs/annie/persistent/users/dschmid/RingCountingStore/models/
+/pnfs/annie/persistent/reconstruction/RingCounting/RingCountingStore/models/
 ```
 
 ## Data
 
 - In the "load_from_file" mode, this tool adds single- and multi-ring (SR/MR) predictions to the RecoEvent BoostStore. When theh "load_from_file" config parameter is set to 0, the tool instead outputs the predictions to a csv file.
 - The predictions are stored in the "RingCountingSRPrediction" and "RingCountingMRPrediction" variables 
 
+The predictions are given as probabilities in the interval [0,1] as doubles, and sum to 1 for both classes.
+
+Implemented as:
+```
+reco_event_bs = self.m_data.Stores.at("RecoEvent")
+
+reco_event_bs.Set("RingCountingSRPrediction", predicted_sr)
+reco_event_bs.Set("RingCountingMRPrediction", predicted_mr)
+```
+
 ---
 ## Configuration
 
-The following configuration variables must be set for the tool to function properly (further details)
-are found at the top of the RingCounting.py file.
+The following configuration variables must be set for the tool to function properly.
+
 ---
 Exemplary configuration:
 ```
+PythonScript RingCounting
+
+InitialiseFunction Initialise
+ExecuteFunction Execute
+FinaliseFunction Finalise
+
+verbose 1
+
 load_from_file 0                                                # If set to 1, load CNNImage formatted csv files
 files_to_load configfiles/RingCounting/files_to_load.txt        # txt file containing files to load in case load_from_file == 1
 version 1_0_0                                                   # Model version
 model_path /annie/app/users/dschmid/RingCountingStore/models/   # Model path
-pmt_mask november_22                                            # PMT mask (zeroed out)
-save_to RC_output.csv                                           # if load_from_file == 1, save predictions as csv
+pmt_mask november_22                                            # Masked PMTs (name of hard-coded set of PMTs to ignore)
+save_to RC_output.csv                                           # Prediction output file, used if load_from_file == 1
 ```

configfiles/CNNImage/CNNImage_Data/CNNImageConfig

@@ -1,12 +1,14 @@
 # CNNImage config file
 
-verbosity 1
-DataMode Normal			#options: Charge/Time
-SaveMode PMT-wise		#options: Geometric / PMT-wise
+verbosity 0
+DataMode Normal			#options: Normal / Charge-Weighted / TimeEvolution
+SaveMode Static			#options: Static / Geometric
 DimensionX 16			#choose something suitable (32/64/...)
-DimensionY 20			#choose something suitable (32/64/...)
-OutputFile R1613p1_CNNImage
-DimensionLAPPD 5              	# Size of the LAPPD pannal 
+DimensionY 10			#choose something suitable (32/64/...)
+OutputFile CNNImage_data_testData
+useLAPPDs 0
+DimensionLAPPD 5              	#Size of the LAPPD pannal
 IncludeTopBottom 1
-DetectorConf ANNIEp2v6		#specify the detector version used in simulation
-IsData 1			#IsData: 0/1
+DetectorConf ANNIEp2v7		#specify the detector version used in simulation
+IsData 1
+WriteToFile 0

configfiles/CNNImage/CNNImage_MC/CNNImageConfig

@@ -1,11 +1,15 @@
 # CNNImage config file
 
-verbosity 0
-DataMode Normal			#options: Charge/Time
-SaveMode PMT-wise		#options: Geometric / PMT-wise
+verbosity 1
+IsData 0
+DataMode Normal			#options: Normal / Charge-Weighted / TimeEvolution
+SaveMode Static		    #options: Static / Geometric
 DimensionX 16			#choose something suitable (32/64/...)
-DimensionY 20			#choose something suitable (32/64/...)
-OutputFile test_beam_lappd5x5_0
-DimensionLAPPD 5              # Size of the LAPPD pannal 
+DimensionY 32			#choose something suitable (32/64/...)
+OutputFile ./test_output_cnnimageNM_geometric
+DimensionLAPPD 5                #Size of the LAPPD pannal
 IncludeTopBottom 1
-DetectorConf ANNIEp2v6		#specify the detector version used in simulation
+DetectorConf ANNIEp2v7		#specify the detector version used in simulation
+useLAPPDs 0
+WriteToFile 1
+

configfiles/RingCounting/CNNImageConfig

@@ -0,0 +1,15 @@
+# CNNImage config file
+
+verbosity 1
+DataMode Normal			#options: Normal / Charge-Weighted / TimeEvolution
+SaveMode Static 		#options: Static / Geometric / PMT-wise
+DimensionX 16			#choose something suitable (32/64/...)
+DimensionY 10			#choose something suitable (32/64/...)
+OutputFile cnnimage_electron_la_sE_1_94
+MCStaticMapping configfiles/CNNImage/mc_cnnimage_mapping.csv
+DataStaticMapping configfiles/CNNImage/data_cnnimage_mapping.csv
+DimensionLAPPD 5                #Size of the LAPPD pannal 
+IncludeTopBottom 1
+DetectorConf ANNIEp2v7		#specify the detector version used in simulation
+useLAPPDs 0
+WriteToFile 0