diff --git a/Framework/include/QualityControl/Check.h b/Framework/include/QualityControl/Check.h index 879808b2cb..2d9b969c29 100644 --- a/Framework/include/QualityControl/Check.h +++ b/Framework/include/QualityControl/Check.h @@ -19,8 +19,10 @@ // O2 #include // QC +#include "QualityControl/DatabaseInterface.h" #include "QualityControl/QualityObject.h" #include "QualityControl/CheckConfig.h" +#include "QualityControl/CheckInterface.h" namespace o2::quality_control::core { @@ -92,6 +94,11 @@ class Check static CheckConfig extractConfig(const core::CommonSpec&, const CheckSpec&); static framework::OutputSpec createOutputSpec(const std::string& detector, const std::string& checkName); + void setDatabase(std::shared_ptr database) + { + mCheckInterface->setDatabase(database); + } + private: void beautify(std::map>& moMap, const core::Quality& quality); diff --git a/Framework/include/QualityControl/CheckInterface.h b/Framework/include/QualityControl/CheckInterface.h index efe63cf29b..b1bb206167 100644 --- a/Framework/include/QualityControl/CheckInterface.h +++ b/Framework/include/QualityControl/CheckInterface.h @@ -17,6 +17,7 @@ #ifndef QC_CHECKER_CHECKINTERFACE_H #define QC_CHECKER_CHECKINTERFACE_H +#include "QualityControl/DatabaseInterface.h" #include "QualityControl/Quality.h" #include "QualityControl/UserCodeInterface.h" #include "QualityControl/Activity.h" @@ -82,10 +83,26 @@ class CheckInterface : public core::UserCodeInterface virtual void startOfActivity(const core::Activity& activity); // not fully abstract because we don't want to change all the existing subclasses virtual void endOfActivity(const core::Activity& activity); // not fully abstract because we don't want to change all the existing subclasses + void setDatabase(std::shared_ptr database) + { + mDatabase = database; + } + protected: /// \brief Called each time mCustomParameters is updated. virtual void configure() override; + /// \brief Retrieve a reference plot at the provided path, matching the give activity and for the provided run. + /// the activity is the current one, while the run number is the reference run. + /// + /// \param path path to the object (no provenance) + /// \param referenceActivity Reference activity (usually a copy of the current activity with a different run number) + /// \return + std::shared_ptr retrieveReference(std::string path, Activity referenceActivity); + + private: + std::shared_ptr mDatabase; + ClassDef(CheckInterface, 6) }; diff --git a/Framework/include/QualityControl/ReferenceUtils.h b/Framework/include/QualityControl/ReferenceUtils.h new file mode 100644 index 0000000000..5b8f6d7b6f --- /dev/null +++ b/Framework/include/QualityControl/ReferenceUtils.h @@ -0,0 +1,47 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file ReferenceComparatorUtils.h +/// \author Andrea Ferrero and Barthelemy von Haller +/// + +#ifndef QUALITYCONTROL_ReferenceUtils_H +#define QUALITYCONTROL_ReferenceUtils_H + +#include +#include "QualityControl/MonitorObject.h" +#include "QualityControl/DatabaseInterface.h" +#include "QualityControl/Activity.h" +#include "QualityControl/ActivityHelpers.h" +#include "QualityControl/QcInfoLogger.h" +#include "QualityControl/RepoPathUtils.h" + +namespace o2::quality_control::checker +{ + +//_________________________________________________________________________________________ +// +// Get the reference plot for a given MonitorObject path + +static std::shared_ptr getReferencePlot(quality_control::repository::DatabaseInterface* qcdb, std::string& fullPath, + core::Activity referenceActivity) +{ + auto [success, path, name] = o2::quality_control::core::RepoPathUtils::splitObjectPath(fullPath); + if (!success) { + return nullptr; + } + return qcdb->retrieveMO(path, name, repository::DatabaseInterface::Timestamp::Latest, referenceActivity); +} + +} // namespace o2::quality_control::checker + +#endif // QUALITYCONTROL_ReferenceUtils_H diff --git a/Framework/include/QualityControl/RepoPathUtils.h b/Framework/include/QualityControl/RepoPathUtils.h index ac03cfeeb8..bc3445bd8c 100644 --- a/Framework/include/QualityControl/RepoPathUtils.h +++ b/Framework/include/QualityControl/RepoPathUtils.h @@ -128,6 +128,36 @@ class RepoPathUtils static constexpr auto allowedProvenancesMessage = R"(Allowed provenances are "qc" (real data processed synchronously), "qc_async" (real data processed asynchronously) and "qc_mc" (simulated data).)"; static bool isProvenanceAllowed(const std::string& provenance); + + /** + * Splits the provided path and returns both the base path and the object name. + * @param fullPath + * @return A tuple with 1. a boolean to specify if we succeeded (i.e. whether we found a `/`) + * 2. the path + * 3. the object name + */ + static std::tuple splitObjectPath(const std::string& fullPath) + { + std::string delimiter = "/"; + std::string det; + size_t pos = fullPath.rfind(delimiter); + if (pos == std::string::npos) { + return { false, "", "" }; + } + std::string path = fullPath.substr(0, pos); + std::string name = fullPath.substr(pos + 1); + return { true, path, name }; + } + + static std::string getPathNoProvenance(std::shared_ptr mo) + { + std::string path = mo->getPath(); + size_t pos = path.find('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } }; } // namespace o2::quality_control::core diff --git a/Framework/src/CheckInterface.cxx b/Framework/src/CheckInterface.cxx index 6d07cb3cda..445d51e609 100644 --- a/Framework/src/CheckInterface.cxx +++ b/Framework/src/CheckInterface.cxx @@ -15,6 +15,7 @@ /// #include "QualityControl/CheckInterface.h" +#include "QualityControl/ReferenceUtils.h" #include "QualityControl/MonitorObject.h" #include @@ -59,4 +60,9 @@ void CheckInterface::endOfActivity(const Activity& activity) // noop, override it if you want. } +shared_ptr CheckInterface::retrieveReference(std::string path, Activity referenceActivity) +{ + return o2::quality_control::checker::getReferencePlot(mDatabase.get(), path, referenceActivity); +} + } // namespace o2::quality_control::checker diff --git a/Framework/src/CheckRunner.cxx b/Framework/src/CheckRunner.cxx index 62cad609c5..e231e9131e 100644 --- a/Framework/src/CheckRunner.cxx +++ b/Framework/src/CheckRunner.cxx @@ -231,6 +231,7 @@ void CheckRunner::init(framework::InitContext& iCtx) updatePolicyManager.reset(); for (auto& [checkName, check] : mChecks) { check.init(); + check.setDatabase(mDatabase); updatePolicyManager.addPolicy(check.getName(), check.getUpdatePolicyType(), check.getObjectsNames(), check.getAllObjectsOption(), false); } } catch (...) { diff --git a/Modules/Common/include/Common/ReferenceComparatorCheck.h b/Modules/Common/include/Common/ReferenceComparatorCheck.h index 1f7f070b25..9e163f9bd2 100644 --- a/Modules/Common/include/Common/ReferenceComparatorCheck.h +++ b/Modules/Common/include/Common/ReferenceComparatorCheck.h @@ -49,9 +49,13 @@ class ReferenceComparatorCheck : public o2::quality_control::checker::CheckInter void endOfActivity(const Activity& activity) override; private: + Quality getSinglePlotQuality(std::shared_ptr mo, std::string& message); + std::unique_ptr mComparator; std::map mQualityFlags; std::map> mQualityLabels; + quality_control::core::Activity mActivity /*current*/, mReferenceActivity; + size_t mReferenceRun; }; } // namespace o2::quality_control_modules::common diff --git a/Modules/Common/include/Common/ReferenceComparatorTask.h b/Modules/Common/include/Common/ReferenceComparatorTask.h index 195f9f27a5..5990f50fec 100644 --- a/Modules/Common/include/Common/ReferenceComparatorTask.h +++ b/Modules/Common/include/Common/ReferenceComparatorTask.h @@ -19,14 +19,11 @@ #define QUALITYCONTROL_REFERENCECOMPARATORTASK_H #include "Common/ReferenceComparatorTaskConfig.h" -#include "Common/BigScreenCanvas.h" #include "QualityControl/PostProcessingInterface.h" #include "QualityControl/DatabaseInterface.h" -#include "QualityControl/Quality.h" #include #include #include -#include #include #include #include @@ -63,8 +60,6 @@ class ReferenceComparatorTask : public quality_control::postprocessing::PostProc }; private: - std::shared_ptr getReferencePlot(o2::quality_control::repository::DatabaseInterface& qcdb, std::string fullPath, o2::quality_control::core::Activity activity); - int mReferenceRun{ 0 }; int mNotOlderThan{ 120 }; diff --git a/Modules/Common/src/ReferenceComparatorCheck.cxx b/Modules/Common/src/ReferenceComparatorCheck.cxx index bde10cf1a3..3800570252 100644 --- a/Modules/Common/src/ReferenceComparatorCheck.cxx +++ b/Modules/Common/src/ReferenceComparatorCheck.cxx @@ -15,8 +15,8 @@ /// #include "Common/ReferenceComparatorCheck.h" +#include "QualityControl/ReferenceUtils.h" #include "Common/TH1Ratio.h" -#include "Common/TH2Ratio.h" #include "QualityControl/MonitorObject.h" #include "QualityControl/Quality.h" #include "QualityControl/QcInfoLogger.h" @@ -25,6 +25,10 @@ #include #include +#include + +#include + // ROOT #include #include @@ -42,9 +46,10 @@ void ReferenceComparatorCheck::configure() void ReferenceComparatorCheck::startOfActivity(const Activity& activity) { - auto moduleName = mCustomParameters.atOptional("moduleName").value_or(""); - auto comparatorName = mCustomParameters.atOptional("comparatorName").value_or(""); - double threshold = std::stof(mCustomParameters.atOptional("threshold").value_or("0")); + auto moduleName = mCustomParameters.atOptional("moduleName", activity).value_or(""); + auto comparatorName = mCustomParameters.atOptional("comparatorName", activity).value_or(""); + double threshold = std::stof(mCustomParameters.atOptional("threshold", activity).value_or("0")); + mReferenceRun = std::stoi(mCustomParameters.atOptional("referenceRun", activity).value_or("0")); mComparator.reset(); if (!moduleName.empty() && !comparatorName.empty()) { @@ -54,6 +59,10 @@ void ReferenceComparatorCheck::startOfActivity(const Activity& activity) if (mComparator) { mComparator->setThreshold(threshold); } + + mActivity = activity; + mReferenceActivity = activity; + mReferenceActivity.mId = mReferenceRun; } void ReferenceComparatorCheck::endOfActivity(const Activity& activity) @@ -129,6 +138,31 @@ static Quality compare(TCanvas* canvas, ObjectComparatorInterface* comparator, s return comparator->compare(plots.first, plots.second, message); } +Quality ReferenceComparatorCheck::getSinglePlotQuality(std::shared_ptr mo, std::string& message) +{ + // retrieve the reference plot and compare + auto* th1 = dynamic_cast(mo->getObject()); + if (th1 == nullptr) { + message = "The MonitorObject is not a TH1"; + return Quality::Null; + } + + // get path of mo and ref (we have to remove the provenance) + std::string path = RepoPathUtils::getPathNoProvenance(mo); + // todo we could cache the reference plot within a run + auto referencePlot = retrieveReference(path, mReferenceActivity); + if (!referencePlot) { + message = "Reference plot not found"; + return Quality::Null; + } + auto* ref = dynamic_cast(referencePlot->getObject()); + if (!ref) { + message = "The reference plot is not a TH1"; + return Quality::Null; + } + return mComparator.get()->compare(th1, ref, message); +} + //_________________________________________________________________________________________ // // Loop over all the input MOs and compare each of them with the corresponding MO from the reference run @@ -136,20 +170,31 @@ static Quality compare(TCanvas* canvas, ObjectComparatorInterface* comparator, s Quality ReferenceComparatorCheck::check(std::map>* moMap) { - Quality result = Quality::Good; + Quality result = Quality::Null; + + if (mReferenceRun == 0) { + result.addFlag(FlagTypeFactory::Unknown(), "No reference run provided"); + return result; + } + for (auto& [key, mo] : *moMap) { auto moName = mo->getName(); + Quality quality; + std::string message; - auto* canvas = dynamic_cast(mo->getObject()); - if (!canvas) { + // run the comparison algorithm + if (mo->getObject()->IsA() == TClass::GetClass()) { + // We got a canvas. It contains the plot and its reference. + auto* canvas = dynamic_cast(mo->getObject()); + quality = compare(canvas, mComparator.get(), message); + } else if (mo->getObject()->InheritsFrom(TH1::Class())) { + // We got a plot, we have to find the reference before calling the comparator + quality = getSinglePlotQuality(mo, message); + } else { + ILOG(Warning, Ops) << "Compared Monitor Object '" << mo->getName() << "' is not a TCanvas or a TH1, the detector QC responsible should review the configuration" << ENDM; continue; } - // run the comparison algorithm - Quality quality; - std::string message; - quality = compare(canvas, mComparator.get(), message); - // update the overall quality if (quality.isWorseThan(result)) { result.set(quality); @@ -191,6 +236,23 @@ static int getQualityColor(const Quality& q) return 0; } +static void updateQualityLabel(TPaveText* label, const Quality& quality) +{ + // draw the quality label with the text color corresponding to the quality level + label->SetTextColor(getQualityColor(quality)); + label->AddText(quality.getName().c_str()); + + // add the first flag below the quality label, or an empty line if no flags are set + auto flags = quality.getFlags(); + std::string message = flags.empty() ? "" : flags.front().second; + auto pos = message.find(" "); + if (pos != std::string::npos) { + message.erase(0, pos + 1); + } + auto* text = label->AddText(message.c_str()); + text->SetTextColor(kGray + 1); +} + // Write the quality level and flags in the existing PaveText inside the canvas static void setQualityLabel(TCanvas* canvas, const Quality& quality) { @@ -209,43 +271,34 @@ static void setQualityLabel(TCanvas* canvas, const Quality& quality) continue; } - // draw the quality label with the text color corresponding to the quality level - label->SetTextColor(getQualityColor(quality)); - label->AddText(quality.getName().c_str()); - - // add the first flag below the quality label, or an empty line if no flags are set - auto flags = quality.getFlags(); - std::string message = flags.empty() ? "" : flags.front().second; - auto pos = message.find(" "); - if (pos != std::string::npos) { - message.erase(0, pos + 1); - } - auto* text = label->AddText(message.c_str()); - text->SetTextColor(kGray + 1); - + updateQualityLabel(label, quality); break; } } void ReferenceComparatorCheck::beautify(std::shared_ptr mo, Quality checkResult) { - auto* canvas = dynamic_cast(mo->getObject()); - if (!canvas) { - return; - } - // get the quality associated to the current MO auto moName = mo->getName(); auto quality = mQualityFlags[moName]; - // retrieve the reference plot from the canvas and set the line color according to the quality - auto plots = getPlotsFromCanvas(canvas); - if (plots.second) { - plots.second->SetLineColor(getQualityColor(quality)); - } + auto* canvas = dynamic_cast(mo->getObject()); + if (canvas) { + // retrieve the reference plot from the canvas and set the line color according to the quality + auto plots = getPlotsFromCanvas(canvas); + if (plots.second) { + plots.second->SetLineColor(getQualityColor(quality)); + } - // draw the quality label on the plot - setQualityLabel(canvas, quality); + // draw the quality label on the plot + setQualityLabel(canvas, quality); + } + auto* th1 = dynamic_cast(mo->getObject()); + if (th1) { + auto* qualityLabel = new TPaveText(0.75, 0.65, 0.98, 0.75, "brNDC"); + updateQualityLabel(qualityLabel, quality); + th1->GetListOfFunctions()->Add(qualityLabel); + } } } // namespace o2::quality_control_modules::common diff --git a/Modules/Common/src/ReferenceComparatorTask.cxx b/Modules/Common/src/ReferenceComparatorTask.cxx index a73fcae83b..6904e571e6 100644 --- a/Modules/Common/src/ReferenceComparatorTask.cxx +++ b/Modules/Common/src/ReferenceComparatorTask.cxx @@ -17,8 +17,7 @@ #include "Common/ReferenceComparatorTask.h" #include "Common/ReferenceComparatorPlot.h" -#include "Common/TH1Ratio.h" -#include "Common/TH2Ratio.h" +#include "QualityControl/ReferenceUtils.h" #include "QualityControl/QcInfoLogger.h" #include "QualityControl/MonitorObject.h" #include "QualityControl/DatabaseInterface.h" @@ -26,8 +25,6 @@ // ROOT #include #include -#include -#include using namespace o2::quality_control::postprocessing; using namespace o2::quality_control::core; @@ -36,48 +33,6 @@ using namespace o2::quality_control; namespace o2::quality_control_modules::common { -static bool splitObjectPath(const std::string& fullPath, std::string& path, std::string& name) -{ - std::string delimiter = "/"; - std::string det; - size_t pos = fullPath.rfind(delimiter); - if (pos == std::string::npos) { - return false; - } - path = fullPath.substr(0, pos); - name = fullPath.substr(pos + 1); - return true; -} - -//_________________________________________________________________________________________ -// -// Helper function for retrieving the last MonitorObject for a give run number - -static std::shared_ptr getMOFromRun(repository::DatabaseInterface* qcdb, const std::string& fullPath, uint32_t run, Activity activity) -{ - ILOG(Info, Devel) << "Loading object '" << fullPath << "' for reference run '" << run << "' and activity " << activity << ENDM; - uint64_t timeStamp = 0; - activity.mId = run; - const auto filterMetadata = activity_helpers::asDatabaseMetadata(activity, false); - const auto objectValidity = qcdb->getLatestObjectValidity(activity.mProvenance + "/" + fullPath, filterMetadata); - if (objectValidity.isValid()) { - timeStamp = objectValidity.getMax() - 1; - } else { - ILOG(Warning, Devel) << "Could not find the object '" << fullPath << "' for reference run " << activity.mId << ENDM; - return nullptr; - } - - std::string path; - std::string name; - if (!splitObjectPath(fullPath, path, name)) { - return nullptr; - } - // retrieve MO from CCDB - auto mo = qcdb->retrieveMO(path, name, timeStamp, activity); - - return mo; -} - //_________________________________________________________________________________________ // Helper function for retrieving a MonitorObject from the QCDB, in the form of a std::pair, bool> // A non-null MO is returned in the first element of the pair if the MO is found in the QCDB @@ -97,9 +52,8 @@ static std::pair, bool> getMO(repository::Databas return { nullptr, false }; } - std::string path; - std::string name; - if (!splitObjectPath(fullPath, path, name)) { + auto [success, path, name] = o2::quality_control::core::RepoPathUtils::splitObjectPath(fullPath); + if (!success) { return { nullptr, false }; } // retrieve QO from CCDB - do not associate to trigger activity if ignoreActivity is true @@ -118,15 +72,6 @@ static std::pair, bool> getMO(repository::Databas return { qo, true }; } -//_________________________________________________________________________________________ -// -// Get the reference plot for a given MonitorObject path - -std::shared_ptr ReferenceComparatorTask::getReferencePlot(repository::DatabaseInterface& qcdb, std::string fullPath, Activity activity) -{ - return getMOFromRun(&qcdb, fullPath, mReferenceRun, activity); -} - //_________________________________________________________________________________________ void ReferenceComparatorTask::configure(const boost::property_tree::ptree& config) @@ -174,7 +119,9 @@ void ReferenceComparatorTask::initialize(quality_control::postprocessing::Trigge auto fullOutPath = group.outputPath + "/" + path; // retrieve the reference MO - auto referencePlot = getReferencePlot(qcdb, fullRefPath, trigger.activity); + auto referenceActivity = trigger.activity; + referenceActivity.mId = mReferenceRun; + auto referencePlot = o2::quality_control::checker::getReferencePlot(&qcdb, fullRefPath, referenceActivity); if (!referencePlot) { continue; } diff --git a/doc/Advanced.md b/doc/Advanced.md index adb81dfc6b..ca6f291bdb 100644 --- a/doc/Advanced.md +++ b/doc/Advanced.md @@ -1398,6 +1398,74 @@ void customize(std::vector& workflowOptions) } ``` +## Reference data + +A reference object is an object from a previous run. It is usually used as a point of comparison. + +### Get a reference plot in a check + +To retrieve a reference plot in your Check, use +``` + std::shared_ptr CheckInterface::retrieveReference(std::string path, Activity referenceActivity); +``` +- `path` : the path of the object _without the provenance (e.g. `qc`)_ +- `referenceActivity` : the activity of reference (usually the current activity with a different run number) + +If the reference is not found it will return a `nullptr` and the quality is `Null`. + +### Compare to a reference plot + +The check `ReferenceComparatorCheck` in `Common` compares objects to their reference. + +The configuration looks like +``` + "QcCheck": { + "active": "true", + "className": "o2::quality_control_modules::common::ReferenceComparatorCheck", + "moduleName": "QcCommon", + "policy": "OnAny", + "detectorName": "TST", + "dataSource": [{ + "type": "Task", + "name": "QcTask", + "MOs": ["example"] + }], + "extendedCheckParameters": { + "default": { + "default": { + "referenceRun" : "500", + "moduleName" : "QualityControl", + "comparatorName" : "o2::quality_control_modules::common::ObjectComparatorChi2", + "threshold" : "0.5" + } + }, + "PHYSICS": { + "PROTON-PROTON": { + "referenceRun" : "551890" + } + } + } + } +``` +The check needs the following parameters +- `referenceRun` to specify what is the run of reference and retrieve the reference data. +- `comparatorName` to decide how to compare, see below for their descriptions. +- `threshold` to specifie the value used to discriminate between good and bad matches between the histograms. + +Three comparators are provided: +1. `o2::quality_control_modules::common::ObjectComparatorDeviation`: comparison based on the average relative deviation between the bins of the current and reference histograms; the `threshold` parameter represent in this case the maximum allowed deviation +2. `o2::quality_control_modules::common::ObjectComparatorChi2`: comparison based on a standard chi2 test between the current and reference histograms; the `threshold` parameter represent in this case the minimum allowed chi2 probability +3. `o2::quality_control_modules::common::ObjectComparatorKolmogorov`: comparison based on a standard Kolmogorov test between the current and reference histograms; the `threshold` parameter represent in this case the minimum allowed Kolmogorov probability + +Note that you can easily specify different reference runs for different run types and beam types. + +The plot is beautified by the addition of a `TPaveText` containing the quality and the reason for the quality. + +### Generate a canvas combining both the current and reference ratio histogram + +The postprocessing task ReferenceComparatorTask draws a given set of plots in comparison with their corresponding references, both as superimposed histograms and as current/reference ratio histograms. +See the details [here](https://github.com/AliceO2Group/QualityControl/blob/master/doc/PostProcessing.md#the-referencecomparatortask-class). + ## Configuration files details The QC requires a number of configuration items. An example config file is