Proposed DataWriter and HDF5DataStore configuration parameters for v2.8.1

12-Oct-2021, KAB

🔺A somewhat important question:🔺 do we want to add the run number somewhere in the HDF5 Group/DataSet path?

This page has the proposed configuration parameters for writing HDF5 files in dunedaq-v2.8.1 (shown below).

Changes to support this mode are on the kbiery/HDF5WriteForColdboxes branches in the dfmodules and minidaqapp repositories.

In these code changes, there are command-line options to mdapp_multrun_gen.py for controlling the output directory and for controlling the file prefix and operational environment.

# relevant mdapp_multiru_gen command-line options
  -o, --output-path PATH          [default: .]

  --op-env TEXT                   Operational environment - used for raw data
                                  filename prefix and HDF5 Attribute inside
                                  the files  [default: swtest]

The "version" number isn't used for anything other than being copied into an HDF5 Attribute. The idea was to give readers of the raw data files an indicator that something changed in the way that the HDF5 Groups and DataSets are named and/or organized. Currently, it can not be set from the mdapp_multru_gen command line, but we could change that, if needed.

# proposed configuration paramters
{
  "data": {
    "data_storage_prescale": 1,  # prescales the writing of the TRs (used only for long DAQ stability tests)
    "data_store_parameters": {
      "directory_path": ".",  # where the raw data files should be written
      "disable_unique_filename_suffix": false,  # disables the "_<timestamp>" string in the filenames
      "file_layout_parameters": {
        "digits_for_trigger_number": 5,
        "path_param_list": [
          {
            "detector_group_name": "TPC",  # name that will appear as an HDF5 Group in the data file
            "detector_group_type": "TPC",  # keyword for set of params; needs to match StorageKey::DataRecordGroupType
            "digits_for_element_number": 2,
            "digits_for_region_number": 3,
            "element_name_prefix": "Link",
            "region_name_prefix": "APA"
          },
          {
            "detector_group_name": "PDS",  # name that will appear as an HDF5 Group in the data file
            "detector_group_type": "PDS",  # keyword for set of params; needs to match StorageKey::DataRecordGroupType
            "digits_for_element_number": 2,
            "digits_for_region_number": 3,
            "element_name_prefix": "Element",
            "region_name_prefix": "Region"
          },
          {
            "detector_group_name": "NDLArTPC",  # name that will appear as an HDF5 Group in the data file
            "detector_group_type": "NDLArTPC",  # keyword for set of params; needs to match StorageKey::DataRecordGroupType
            "digits_for_element_number": 2,
            "digits_for_region_number": 3,
            "element_name_prefix": "Element",
            "region_name_prefix": "Region"
          },
          {
            "detector_group_name": "Trigger",  # name that will appear as an HDF5 Group in the data file
            "detector_group_type": "Trigger",  # keyword for set of params; needs to match StorageKey::DataRecordGroupType
            "digits_for_element_number": 2,
            "digits_for_region_number": 3,
            "element_name_prefix": "Element",
            "region_name_prefix": "Region"
          },
          {
            "detector_group_name": "TPC",  # name that will appear as an HDF5 Group in the data file
            "detector_group_type": "TPC_TP",  # keyword for set of params; needs to match StorageKey::DataRecordGroupType
            "digits_for_element_number": 2,
            "digits_for_region_number": 3,
            "element_name_prefix": "Link",
            "region_name_prefix": "TP_APA"
          }
        ],
        "trigger_record_name_prefix": "TriggerRecord"
      },
      "filename_parameters": {
        # these parameters result in filenames like swtest_run123456_0000<optional timestamp>.hdf5
        "digits_for_file_index": 4,
        "digits_for_run_number": 6,
        "file_index_prefix": "",
        "overall_prefix": "swtest",
        "run_number_prefix": "run"
      },
      "free_space_safety_factor_for_write": 5.0,  # used to check for a full disk when writing a TR
      "max_file_size_bytes": 1073741824,
      "mode": "all-per-file",  # historical data-storage mode, should always be "all-per-file" these days
      "name": "data_store",
      "operational_environment": "swtest",  # stored as an HDF5 Attribute inside the raw data file
      "type": "HDF5DataStore",
      "version": 3  # stored as an HDF5 Attribute inside the raw data file
    },
    "initial_token_count": 0,
    "max_write_retry_time_usec": 1000000,
    "min_write_retry_time_usec": 1000,
    "write_retry_time_increase_factor": 2
  },
  "match": "datawriter"
},