From 276537feaa581956561ac9b81cb1c6aa0b3b1144 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Fri, 30 Aug 2024 17:06:42 +0200 Subject: [PATCH] QC-604 removed analysis integration except of QCDB upload The possibility to consume AODs and generating DPL jsons to run in the train system proved not to be used and needed in the end. `o2-qc-upload-root-objects` remains the only allowed interface between Analysis and QC. --- CMakeLists.txt | 1 - Modules/Example/CMakeLists.txt | 20 - Modules/Example/etc/analysisDerived.json | 54 - Modules/Example/etc/analysisDirect.json | 54 - .../Example/include/Example/AnalysisTask.h | 54 - Modules/Example/include/Example/LinkDef.h | 1 - .../o2-qc-example-analysis-derived.json | 936 ------------------ .../o2-qc-example-analysis-direct.json | 936 ------------------ Modules/Example/src/AnalysisTask.cxx | 96 -- README.md | 3 - cmake/AddQCWorkflow.cmake | 52 - doc/Advanced.md | 106 +- 12 files changed, 2 insertions(+), 2311 deletions(-) delete mode 100644 Modules/Example/etc/analysisDerived.json delete mode 100644 Modules/Example/etc/analysisDirect.json delete mode 100644 Modules/Example/include/Example/AnalysisTask.h delete mode 100644 Modules/Example/o2-qc-example-analysis-derived.json delete mode 100644 Modules/Example/o2-qc-example-analysis-direct.json delete mode 100644 Modules/Example/src/AnalysisTask.cxx delete mode 100644 cmake/AddQCWorkflow.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 03f6a5ad98..f95f77dbab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,6 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) include(CMakeParseArguments) include(GNUInstallDirs) -include(AddQCWorkflow) # ---- Project ---- diff --git a/Modules/Example/CMakeLists.txt b/Modules/Example/CMakeLists.txt index 85d9aa20e0..cc93eb287b 100644 --- a/Modules/Example/CMakeLists.txt +++ b/Modules/Example/CMakeLists.txt @@ -4,7 +4,6 @@ add_library(O2QcExample src/BenchmarkTask.cxx src/EveryObject.cxx src/ExampleTask.cxx - src/AnalysisTask.cxx src/FakeCheck.cxx src/ExampleCondition.cxx src/CustomTH2F.cxx @@ -21,7 +20,6 @@ target_link_libraries(O2QcExample PUBLIC O2QualityControl) add_root_dictionary(O2QcExample HEADERS include/Example/BenchmarkTask.h include/Example/EveryObject.h - include/Example/AnalysisTask.h include/Example/ExampleTask.h include/Example/FakeCheck.h include/Example/ExampleCondition.h @@ -36,24 +34,6 @@ install(TARGETS O2QcExample install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/Example DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/QualityControl") -# ---- Workflows for analysis ---- - -# This is disabled because it can create problems when building on an FLP. This is due to the fact that we use the install directory before the install stage. - -# Install the config files for the analysis. -# The command "install" cannot be used because it would be executed after o2_add_qc_workflow which relies on -# the config files being already installed in the right place. -# Note: There is an issue with ninja when a "configured" file is deleted. https://gitlab.kitware.com/cmake/cmake/-/issues/18985 -# The set_property should fix it. -#configure_file("etc/analysisDirect.json" "${CMAKE_INSTALL_PREFIX}/etc/analysisDirect.json") -#set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_INSTALL_PREFIX}/etc/analysisDirect.json") -#configure_file("etc/analysisDerived.json" "${CMAKE_INSTALL_PREFIX}/etc/analysisDerived.json") -#set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_INSTALL_PREFIX}/etc/analysisDerived.json") -# -## We cannot point to the json files in the source dir because the path will be reused in the produced json. -#o2_add_qc_workflow(WORKFLOW_NAME o2-qc-example-analysis-direct CONFIG_FILE_PATH ${CMAKE_INSTALL_PREFIX}/etc/analysisDirect.json) -#o2_add_qc_workflow(WORKFLOW_NAME o2-qc-example-analysis-derived CONFIG_FILE_PATH ${CMAKE_INSTALL_PREFIX}/etc/analysisDerived.json) - # ---- Tests ---- set(TEST_SRCS test/testFactory.cxx test/testQcExample.cxx) diff --git a/Modules/Example/etc/analysisDerived.json b/Modules/Example/etc/analysisDerived.json deleted file mode 100644 index 5b477d2e97..0000000000 --- a/Modules/Example/etc/analysisDerived.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "NONE" - }, - "monitoring": { - "url": "infologger:///debug?qc" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - } - }, - "tasks": { - "AnalysisQcTask": { - "active": "true", - "className": "o2::quality_control_modules::example::AnalysisTask", - "moduleName": "QcExample", - "detectorName": "TST", - "cycleDurationSeconds": "10", - "dataSource": { - "type": "direct", - "query": "aod-data:AOD/HASH/0" - }, - "location": "remote" - } - }, - "checks": { - "DummyAnalysisCheck": { - "active": "true", - "className": "o2::quality_control_modules::skeleton::SkeletonCheck", - "moduleName": "QcSkeleton", - "policy": "OnAny", - "detectorName": "TST", - "dataSource": [{ - "type": "Task", - "name": "AnalysisQcTask", - "MOs": ["example"] - }] - } - } - } -} diff --git a/Modules/Example/etc/analysisDirect.json b/Modules/Example/etc/analysisDirect.json deleted file mode 100644 index 01ed25653f..0000000000 --- a/Modules/Example/etc/analysisDirect.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "NONE" - }, - "monitoring": { - "url": "infologger:///debug?qc" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - } - }, - "tasks": { - "AODQcTask": { - "active": "true", - "className": "o2::quality_control_modules::example::AnalysisTask", - "moduleName": "QcExample", - "detectorName": "TST", - "cycleDurationSeconds": "10", - "dataSource": { - "type": "direct", - "query": "aod-data:AOD/TRACK:PAR/0" - }, - "location": "remote" - } - }, - "checks": { - "DummyAnalysisCheck": { - "active": "true", - "className": "o2::quality_control_modules::skeleton::SkeletonCheck", - "moduleName": "QcSkeleton", - "policy": "OnAny", - "detectorName": "TST", - "dataSource": [{ - "type": "Task", - "name": "AODQcTask", - "MOs": ["example"] - }] - } - } - } -} diff --git a/Modules/Example/include/Example/AnalysisTask.h b/Modules/Example/include/Example/AnalysisTask.h deleted file mode 100644 index 40e529538a..0000000000 --- a/Modules/Example/include/Example/AnalysisTask.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file AnalysisTask.h -/// \author Piotr Konopka -/// - -#ifndef QC_MODULE_EXAMPLE_EXAMPLEANALYSISTASK_H -#define QC_MODULE_EXAMPLE_EXAMPLEANALYSISTASK_H - -#include "QualityControl/TaskInterface.h" - -class TH1F; - -using namespace o2::quality_control::core; - -namespace o2::quality_control_modules::example -{ - -/// \brief An example of QC Task which consumes AODs -/// \author Piotr Konopka -class AnalysisTask final : public quality_control::core::TaskInterface -{ - public: - /// \brief Constructor - AnalysisTask() = default; - /// Destructor - ~AnalysisTask() override; - - // Definition of the methods for the template method pattern - void initialize(o2::framework::InitContext& ctx) override; - void startOfActivity(const Activity& activity) override; - void startOfCycle() override; - void monitorData(o2::framework::ProcessingContext& ctx) override; - void endOfCycle() override; - void endOfActivity(const Activity& activity) override; - void reset() override; - - private: - TH1F* mHistogram = nullptr; -}; - -} // namespace o2::quality_control_modules::example - -#endif // QC_MODULE_EXAMPLE_EXAMPLEANALYSISTASK_H diff --git a/Modules/Example/include/Example/LinkDef.h b/Modules/Example/include/Example/LinkDef.h index cb49469443..5e378ccb21 100644 --- a/Modules/Example/include/Example/LinkDef.h +++ b/Modules/Example/include/Example/LinkDef.h @@ -8,6 +8,5 @@ #pragma link C++ class o2::quality_control_modules::example::BenchmarkTask + ; #pragma link C++ class o2::quality_control_modules::example::ExampleCondition + ; #pragma link C++ class o2::quality_control_modules::example::CustomTH2F + ; -#pragma link C++ class o2::quality_control_modules::example::AnalysisTask + ; #pragma link C++ class o2::quality_control_modules::example::EveryObject+; #endif diff --git a/Modules/Example/o2-qc-example-analysis-derived.json b/Modules/Example/o2-qc-example-analysis-derived.json deleted file mode 100644 index 81a90817b2..0000000000 --- a/Modules/Example/o2-qc-example-analysis-derived.json +++ /dev/null @@ -1,936 +0,0 @@ -{ - "workflow": [ - { - "name": "QC-TASK-RUNNER-AnalysisQcTask", - "inputs": [ - { - "binding": "aod-data", - "origin": "AOD", - "description": "HASH", - "subspec": 0, - "lifetime": 0 - }, - { - "binding": "timer-cycle", - "origin": "QC", - "description": "TIMER-Analysi-mo", - "subspec": 0, - "lifetime": 4 - } - ], - "outputs": [ - { - "binding": "mo", - "origin": "QC", - "description": "AnalysisQcTas-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [ - { - "name": "period-timer-cycle", - "type": "0", - "defaultValue": "10000000", - "help": "timer period" - } - ], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "QC-CHECK-RUNNER-DummyAnalysisCheck", - "inputs": [ - { - "binding": "AnalysisQcTask", - "origin": "QC", - "description": "AnalysisQcTas-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "outputs": [ - { - "binding": "QC/DummyAnalysi-chk/0", - "origin": "QC", - "description": "DummyAnalysi-chk", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - } - ], - "metadata": [ - { - "name": "internal-dpl-clock", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_internal-dpl-aod-reader", - "from_internal-dpl-clock_to_QC-TASK-RUNNER-AnalysisQcTask" - ] - }, - { - "name": "internal-dpl-aod-reader", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_internal-dpl-aod-reader", - "from_internal-dpl-aod-reader_to_internal-dpl-injected-dummy-sink", - "from_internal-dpl-aod-reader_to_QC-TASK-RUNNER-AnalysisQcTask" - ] - }, - { - "name": "QC-TASK-RUNNER-AnalysisQcTask", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_QC-TASK-RUNNER-AnalysisQcTask", - "from_internal-dpl-aod-reader_to_QC-TASK-RUNNER-AnalysisQcTask", - "from_QC-TASK-RUNNER-AnalysisQcTask_to_QC-CHECK-RUNNER-DummyAnalysisCheck" - ] - }, - { - "name": "QC-CHECK-RUNNER-DummyAnalysisCheck", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_QC-TASK-RUNNER-AnalysisQcTask_to_QC-CHECK-RUNNER-DummyAnalysisCheck", - "from_QC-CHECK-RUNNER-DummyAnalysisCheck_to_internal-dpl-global-binary-file-sink" - ] - }, - { - "name": "internal-dpl-injected-dummy-sink", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-aod-reader_to_internal-dpl-injected-dummy-sink" - ] - }, - { - "name": "internal-dpl-global-binary-file-sink", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDerived.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-derived.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_QC-CHECK-RUNNER-DummyAnalysisCheck_to_internal-dpl-global-binary-file-sink" - ] - } - ] -} \ No newline at end of file diff --git a/Modules/Example/o2-qc-example-analysis-direct.json b/Modules/Example/o2-qc-example-analysis-direct.json deleted file mode 100644 index 8ba9846c34..0000000000 --- a/Modules/Example/o2-qc-example-analysis-direct.json +++ /dev/null @@ -1,936 +0,0 @@ -{ - "workflow": [ - { - "name": "QC-TASK-RUNNER-AODQcTask", - "inputs": [ - { - "binding": "aod-data", - "origin": "AOD", - "description": "TRACK:PAR", - "subspec": 0, - "lifetime": 0 - }, - { - "binding": "timer-cycle", - "origin": "QC", - "description": "TIMER-AODQcTa-mo", - "subspec": 0, - "lifetime": 4 - } - ], - "outputs": [ - { - "binding": "mo", - "origin": "QC", - "description": "AODQcTask-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [ - { - "name": "period-timer-cycle", - "type": "0", - "defaultValue": "10000000", - "help": "timer period" - } - ], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "QC-CHECK-RUNNER-DummyAnalysisCheck", - "inputs": [ - { - "binding": "AODQcTask", - "origin": "QC", - "description": "AODQcTask-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "outputs": [ - { - "binding": "QC/DummyAnalysi-chk/0", - "origin": "QC", - "description": "DummyAnalysi-chk", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - } - ], - "metadata": [ - { - "name": "internal-dpl-clock", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_internal-dpl-aod-reader", - "from_internal-dpl-clock_to_QC-TASK-RUNNER-AODQcTask" - ] - }, - { - "name": "internal-dpl-aod-reader", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_internal-dpl-aod-reader", - "from_internal-dpl-aod-reader_to_internal-dpl-injected-dummy-sink", - "from_internal-dpl-aod-reader_to_QC-TASK-RUNNER-AODQcTask" - ] - }, - { - "name": "QC-TASK-RUNNER-AODQcTask", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-clock_to_QC-TASK-RUNNER-AODQcTask", - "from_internal-dpl-aod-reader_to_QC-TASK-RUNNER-AODQcTask", - "from_QC-TASK-RUNNER-AODQcTask_to_QC-CHECK-RUNNER-DummyAnalysisCheck" - ] - }, - { - "name": "QC-CHECK-RUNNER-DummyAnalysisCheck", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_QC-TASK-RUNNER-AODQcTask_to_QC-CHECK-RUNNER-DummyAnalysisCheck", - "from_QC-CHECK-RUNNER-DummyAnalysisCheck_to_internal-dpl-global-binary-file-sink" - ] - }, - { - "name": "internal-dpl-injected-dummy-sink", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_internal-dpl-aod-reader_to_internal-dpl-injected-dummy-sink" - ] - }, - { - "name": "internal-dpl-global-binary-file-sink", - "executable": "../../bin/o2-qc", - "cmdLineArgs": [ - "-b", - "--config", - "json:///home/taillepied/alice/sw/slc7_x86-64/QualityControl/master-1/etc/analysisDirect.json", - "--dump-workflow", - "--dump-workflow-file", - "/home/taillepied/alice/sw/SOURCES/QualityControl/master/0/Modules/Example/o2-qc-example-analysis-direct.json" - ], - "workflowOptions": [ - { - "name": "config", - "type": "4", - "defaultValue": "", - "help": "Path to QC and Data Sampling configuration file." - }, - { - "name": "local", - "type": "5", - "defaultValue": "0", - "help": "Creates only the local part of the QC topology." - }, - { - "name": "host", - "type": "4", - "defaultValue": "", - "help": "Name of the host of the local part of the QC topology.Necessary to specify when creating topologies on multiple machines. If not specified, hostname of the current machine will be used" - }, - { - "name": "remote", - "type": "5", - "defaultValue": "0", - "help": "Creates only the remote part of the QC topology." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Do not add Data Sampling infrastructure." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - }, - { - "name": "clone", - "type": "4", - "defaultValue": "", - "help": "clone processors from a template" - }, - { - "name": "workflow-suffix", - "type": "4", - "defaultValue": "", - "help": "suffix to add to all dataprocessors" - }, - { - "name": "aod-memory-rate-limit", - "type": "1", - "defaultValue": "0", - "help": "Rate limit AOD processing based on memory" - }, - { - "name": "aod-writer-json", - "type": "4", - "defaultValue": "", - "help": "Name of the json configuration file" - }, - { - "name": "aod-writer-resfile", - "type": "4", - "defaultValue": "", - "help": "Default name of the output file" - }, - { - "name": "aod-writer-resmode", - "type": "4", - "defaultValue": "RECREATE", - "help": "Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE" - }, - { - "name": "aod-writer-ntfmerge", - "type": "0", - "defaultValue": "-1", - "help": "Number of time frames to merge into one file" - }, - { - "name": "aod-writer-keep", - "type": "4", - "defaultValue": "", - "help": "Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename" - }, - { - "name": "fairmq-rate-logging", - "type": "0", - "defaultValue": "0", - "help": "Rate logging for FairMQ channels" - }, - { - "name": "fairmq-recv-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "recvBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-send-buffer-size", - "type": "0", - "defaultValue": "1000", - "help": "sendBufferSize option for FairMQ channels" - }, - { - "name": "fairmq-ipc-prefix", - "type": "4", - "defaultValue": "/tmp", - "help": "Prefix for FairMQ channels location" - }, - { - "name": "forwarding-policy", - "type": "4", - "defaultValue": "dangling", - "help": "Which messages to forward. dangling: dangling outputs, all: all messages" - }, - { - "name": "forwarding-destination", - "type": "4", - "defaultValue": "file", - "help": "Destination for forwarded messages. file: write to file, fairmq: send to output proxy" - } - ], - "channels": [ - "from_QC-CHECK-RUNNER-DummyAnalysisCheck_to_internal-dpl-global-binary-file-sink" - ] - } - ] -} \ No newline at end of file diff --git a/Modules/Example/src/AnalysisTask.cxx b/Modules/Example/src/AnalysisTask.cxx deleted file mode 100644 index 1fa04f08aa..0000000000 --- a/Modules/Example/src/AnalysisTask.cxx +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file AnalysisTask.cxx -/// \author Piotr Konopka -/// - -#include -#include - -#include "QualityControl/QcInfoLogger.h" -#include "Example/AnalysisTask.h" -#include -#include -#include - -namespace o2::quality_control_modules::example -{ - -AnalysisTask::~AnalysisTask() -{ - delete mHistogram; -} - -void AnalysisTask::initialize(o2::framework::InitContext& /*ctx*/) -{ - ILOG(Debug, Devel) << "initialize AnalysisTask" << ENDM; // QcInfoLogger is used. FairMQ logs will go to there as well. - - mHistogram = new TH1F("example", "example", 20, 0, 30000); - getObjectsManager()->startPublishing(mHistogram, PublicationPolicy::Forever); -} - -void AnalysisTask::startOfActivity(const Activity& activity) -{ - ILOG(Debug, Devel) << "startOfActivity" << activity.mId << ENDM; - mHistogram->Reset(); -} - -void AnalysisTask::startOfCycle() -{ - ILOG(Debug, Devel) << "startOfCycle" << ENDM; -} - -void AnalysisTask::monitorData(o2::framework::ProcessingContext& ctx) -{ - ILOG(Info, Devel) << "Monitor data" << ENDM; - auto s = ctx.inputs().get("aod-data"); - - auto table = s->asArrowTable(); - if (table->num_rows() == 0) { - ILOG(Error, Support) << "The arrow table is empty" << table->num_rows() << ENDM; - return; - } - ILOG(Info, Devel) << "The arrow table has " << table->num_rows() << " rows" << ENDM; - mHistogram->Fill(table->num_rows()); - - if (table->num_columns() == 0) { - ILOG(Error, Support) << "No columns in the arrow table" << table->num_columns() << ENDM; - return; - } - ILOG(Info, Support) << "The arrow table has " << table->num_columns() << " columns" << ENDM; - mHistogram->Fill(table->num_columns()); - - // Here you can perform analysis of the columnar data. - // Please refer to the documentation of DPL Analysis, Apache Arrow - // and RDataFrame's support of Apache Arrow. -} - -void AnalysisTask::endOfCycle() -{ - ILOG(Debug, Devel) << "endOfCycle" << ENDM; -} - -void AnalysisTask::endOfActivity(const Activity& /*activity*/) -{ - ILOG(Debug, Devel) << "endOfActivity" << ENDM; -} - -void AnalysisTask::reset() -{ - // clean all the monitor objects here - - ILOG(Debug, Devel) << "Resetting the histograms" << ENDM; - mHistogram->Reset(); -} - -} // namespace o2::quality_control_modules::example diff --git a/README.md b/README.md index da588bf36a..5a9f3a3ac0 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,6 @@ For a general overview of our (O2) software, organization and processes, please * [Critical, resilient and non-critical tasks](doc/Advanced.md#critical-resilient-and-non-critical-tasks) * [QC with DPL Analysis](doc/Advanced.md#qc-with-dpl-analysis) * [Uploading objects to QCDB](doc/Advanced.md#uploading-objects-to-qcdb) - * [Getting AODs in QC Tasks](doc/Advanced.md#getting-aods-in-qc-tasks) - * [Merging with other analysis workflows](doc/Advanced.md#merging-with-other-analysis-workflows) - * [Enabling a workflow to run on Hyperloop](doc/Advanced.md#enabling-a-workflow-to-run-on-hyperloop) * [Solving performance issues](doc/Advanced.md#solving-performance-issues) * [Dispatcher](doc/Advanced.md#dispatcher) * [QC Tasks](doc/Advanced.md#qc-tasks-1) diff --git a/cmake/AddQCWorkflow.cmake b/cmake/AddQCWorkflow.cmake deleted file mode 100644 index 72144aa76d..0000000000 --- a/cmake/AddQCWorkflow.cmake +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright CERN and copyright holders of ALICE O2. This software is distributed -# under the terms of the GNU General Public License v3 (GPL Version 3), copied -# verbatim in the file "COPYING". -# -# See http://alice-o2.web.cern.ch/license for full licensing information. -# -# In applying this license CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization or -# submit itself to any jurisdiction. - -include_guard() - -# -# o2_add_qc_workflow(workflow-name configuration-file SOURCES ...) generates -# a configuration json file via o2-qc --config --dump -# so that it can be easily registered and deployed e.g. in the train infrastructure. -# -# The installed JSON file will be named .json and installed -# under share/dpl directory -# - -function(o2_add_qc_workflow) - - cmake_parse_arguments( - PARSED_ARGS # prefix of output variables - "" # list of names of the boolean arguments (only defined ones will be true) - "WORKFLOW_NAME;CONFIG_FILE_PATH" # list of names of mono-valued arguments - "" # list of names of multi-valued arguments (output variables are lists) - ${ARGN} # arguments of the function to parse, here we take the all original ones - ) - # note: if it remains unparsed arguments, here, they can be found in variable PARSED_ARGS_UNPARSED_ARGUMENTS - if(NOT PARSED_ARGS_WORKFLOW_NAME OR NOT PARSED_ARGS_CONFIG_FILE_PATH) - message(FATAL_ERROR "You must provide a workflow name and a config file path") - endif() - set(jsonDumpFile ${CMAKE_CURRENT_BINARY_DIR}/${PARSED_ARGS_WORKFLOW_NAME}.json) - message(STATUS "o2_add_qc_workflow called with config file ${PARSED_ARGS_CONFIG_FILE_PATH} to generate ${jsonDumpFile}") - set(qcExecutable o2-qc) - - # note: we ignore failures because of past problems in the CI. - add_custom_command( - OUTPUT ${jsonDumpFile} - VERBATIM - COMMAND echo "${qcExecutable} -b --config json://${PARSED_ARGS_CONFIG_FILE_PATH} --dump-workflow --dump-workflow-file ${jsonDumpFile} || touch ${jsonDumpFile}" - COMMAND ${qcExecutable} -b --config json://${PARSED_ARGS_CONFIG_FILE_PATH} --dump-workflow --dump-workflow-file ${jsonDumpFile} || touch ${jsonDumpFile} - DEPENDS ${qcExecutable} ${PARSED_ARGS_CONFIG_FILE_PATH}) - - get_filename_component(filename ${jsonDumpFile} NAME_WE) - add_custom_target(${filename} ALL DEPENDS ${jsonDumpFile}) - - install(FILES ${jsonDumpFile} DESTINATION ${CMAKE_INSTALL_DATADIR}/dpl) - -endfunction() \ No newline at end of file diff --git a/doc/Advanced.md b/doc/Advanced.md index e89911c696..3f4690c4dc 100644 --- a/doc/Advanced.md +++ b/doc/Advanced.md @@ -22,9 +22,6 @@ Advanced topics * [Critical, resilient and non-critical tasks](#critical-resilient-and-non-critical-tasks) * [QC with DPL Analysis](#qc-with-dpl-analysis) * [Uploading objects to QCDB](#uploading-objects-to-qcdb) - * [Getting AODs in QC Tasks](#getting-aods-in-qc-tasks) - * [Merging with other analysis workflows](#merging-with-other-analysis-workflows) - * [Enabling a workflow to run on Hyperloop](#enabling-a-workflow-to-run-on-hyperloop) * [Solving performance issues](#solving-performance-issues) * [Dispatcher](#dispatcher) * [QC Tasks](#qc-tasks-1) @@ -557,16 +554,6 @@ If they are not critical, they will be `expendable` and will not stop the run if ## QC with DPL Analysis -QC offers several ways to interact with the DPL Analysis framework. -One allows to [upload root objects generated](#uploading-objects-to-qcdb) by an Analysis Task into QCDB. -It is also possible to [run QC workflows alongside Analysis Tasks](#merging-with-other-analysis-workflows), - as they are also based on the Data Processing Layer. -AOD tables can be [requested as direct data sources](#getting-aods-in-qc-tasks) and then read by a QC task with TableConsumer. - -In this piece of documentation it is assumed that the users already have some idea about QC and -[DPL Analysis](https://aliceo2group.github.io/analysis-framework), and -they have an access to AOD files following the Run 3 data model. - ### Uploading objects to QCDB To upload objects written to a file by an Analysis Task to QCDB, one may use the following command: @@ -595,97 +582,8 @@ the directories listed in the logs: 2021-10-05 10:59:41.594386 Storing MonitorObject qc_mc/TST/MO/AnalysisFromFileTest/hTimeT0Call 2021-10-05 10:59:41.597743 Successfully uploaded 10 objects to the QCDB. ``` -Notice that the executable will ignore the directory structure in the input file and upload all objects to one directory. -If you need a different behaviour, please contact the developers. - -### Getting AODs in QC Tasks - -First, let's see how to get data directly from an AOD file. To read the table, we will use TableConsumer from DPL, as in [the example of a QC analysis -task](../Modules/Example/src/AnalysisTask.cxx): -``` -void AnalysisTask::monitorData(o2::framework::ProcessingContext& ctx) -{ - auto s = ctx.inputs().get("aod-data"); - auto table = s->asArrowTable(); - ... -} -``` -In [our QC configuration file](../Modules/Example/etc/analysisDirect.json) we will request AOD data as a direct source -, just as normal Analysis Tasks do: -``` - "tasks": { - "AnalysisQcTask": { - "active": "true", - "className": "o2::quality_control_modules::example::AnalysisTask", - "moduleName": "QcExample", - "detectorName": "TST", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "aod-data:AOD/TRACK:PAR/0" - }, - "location": "remote" - } - }, -``` -Then we can run the processing with the following command: -``` -o2-qc --config json://${QUALITYCONTROL_ROOT}/etc/analysisDirect.json -b --aod-file AO2D.root -``` - -### Merging with other analysis workflows - -Now, let's try to subscribe to data generated in another analysis workflow - -[`o2-analysistutorial-tracks-combinations`](https://github.com/AliceO2Group/AliceO2/tree/dev/Analysis/Tutorials/src/tracksCombinations.cxx), -which produces a new table with hash numbers generated out of tracks in AODs: -``` -... -DECLARE_SOA_TABLE(Hashes, "AOD", "HASH", hash::Bin); -... -``` -Thus, in [our QC config file](../Modules/Example/etc/analysisDerived.json) we should query data described as `AOD/HASH/0`: -``` - "tasks": { - "AnalysisQcTask": { - "active": "true", - "className": "o2::quality_control_modules::example::AnalysisTask", - "moduleName": "QcExample", - "detectorName": "TST", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "aod-data:AOD/HASH/0" - }, - "location": "remote" - } - }, -``` - -Finally, we can run the example by merging the two workflows. Remember to specify the AOD file path in both workflows -, even if QC does need data directly from the file. -``` -o2-analysistutorial-tracks-combinations --aod-file AO2D.root -b | \ - o2-qc --config json://${QUALITYCONTROL_ROOT}/etc/analysisDerived.json -b --aod-file AO2D.root -``` - -### Enabling a workflow to run on Hyperloop - -Hyperloop requires a workflow JSON dump in order to run it on Grid. To generate such a dump, in CMakeLists.txt of a -detector libraryone should use `configure_file` to install the configuration files, then `o2_add_qc_workflow` to -declare a QC analysis workflow. The first argument is an arbitrary workflow name, the second is the configuration -file path in the installation directory. For example: - -``` -configure_file("etc/analysisDirect.json" "${CMAKE_INSTALL_PREFIX}/etc/analysisDirect.json") -configure_file("etc/analysisDerived.json" "${CMAKE_INSTALL_PREFIX}/etc/analysisDerived.json") - -# ---- Workflows for analysis ---- - -o2_add_qc_workflow(WORKFLOW_NAME o2-qc-example-analysis-direct CONFIG_FILE_PATH ${CMAKE_INSTALL_PREFIX}/etc/analysisDirect.json) -o2_add_qc_workflow(WORKFLOW_NAME o2-qc-example-analysis-derived CONFIG_FILE_PATH ${CMAKE_INSTALL_PREFIX}/etc/analysisDerived.json) -``` +Notice that by default the executable will ignore the directory structure in the input file and upload all objects to one directory. +If you need the directory structure preserved, add the argument `--preserve-directories`. # Solving performance issues