Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ITSMFT DeadMapBuilder ccdb upload when EoS not processed #12828

Merged
merged 6 commits into from
Mar 12, 2024
Merged

ITSMFT DeadMapBuilder ccdb upload when EoS not processed #12828

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b7639fa
[WIP] ITSMFT DeadMapBuilder severity message reduced
nicolovalle Mar 7, 2024
906041b
ccdb api in the workflow
nicolovalle Mar 8, 2024
99593a5
leaving option for offline processing
nicolovalle Mar 9, 2024
cd9b474
dummy commit to trigger CI
nicolovalle Mar 10, 2024
690f57f
ec not nullptr
nicolovalle Mar 11, 2024
3db163e
Dummy commit to trigger CI
nicolovalle Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class ITSMFTDeadMapBuilder : public Task

long mTimeStart = -1; // TODO: better to use RCT info?

std::string mCCDBUrl = "";
std::string mObjectName;
std::string mLocalOutputDir;

Expand All @@ -102,6 +103,7 @@ class ITSMFTDeadMapBuilder : public Task
std::vector<uint16_t> mDeadMapTF{};

unsigned long mFirstOrbitTF = 0x0;
unsigned long mFirstOrbitRun = 0x0;

std::string mDataSource = "chipsstatus";

Expand All @@ -110,7 +112,7 @@ class ITSMFTDeadMapBuilder : public Task
o2::itsmft::TimeDeadMap mMapObject;

void finalizeOutput();
void PrepareOutputCcdb(DataAllocator& output);
void PrepareOutputCcdb(EndOfStreamContext* ec, std::string ccdburl);

// Utils

Expand All @@ -121,9 +123,6 @@ class ITSMFTDeadMapBuilder : public Task

// Flag to avoid that endOfStream and stop are both done
bool isEnded = false;

// Run stop requested flag for EoS operations
bool mRunStopRequested = false;
};

// Create a processor spec
Expand Down
66 changes: 45 additions & 21 deletions Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ void ITSMFTDeadMapBuilder::init(InitContext& ic)
mTFLength = ic.options().get<int>("tf-length");
mDoLocalOutput = ic.options().get<bool>("local-output");
mObjectName = ic.options().get<std::string>("outfile");
mCCDBUrl = ic.options().get<std::string>("ccdb-url");
nicolovalle marked this conversation as resolved.
Show resolved Hide resolved
mLocalOutputDir = ic.options().get<std::string>("output-dir");
mSkipStaticMap = ic.options().get<bool>("skip-static-map");

Expand Down Expand Up @@ -131,9 +132,6 @@ void ITSMFTDeadMapBuilder::finalizeOutput()
// Main running function
void ITSMFTDeadMapBuilder::run(ProcessingContext& pc)
{
if (mRunStopRequested) { // give up when run stop request arrived
return;
}

std::chrono::time_point<std::chrono::high_resolution_clock> start;
std::chrono::time_point<std::chrono::high_resolution_clock> end;
Expand All @@ -144,7 +142,11 @@ void ITSMFTDeadMapBuilder::run(ProcessingContext& pc)

mFirstOrbitTF = pc.services().get<o2::framework::TimingInfo>().firstTForbit;

if ((unsigned long)(mFirstOrbitTF / mTFLength) % mTFSampling != 0) {
if (mFirstOrbitRun == 0x0) {
mFirstOrbitRun = mFirstOrbitTF;
}

if ((((long)mFirstOrbitTF - mFirstOrbitRun) / mTFLength) % mTFSampling != 0) {
return;
}

Expand Down Expand Up @@ -235,9 +237,11 @@ void ITSMFTDeadMapBuilder::run(ProcessingContext& pc)
}

//////////////////////////////////////////////////////////////////////////////
void ITSMFTDeadMapBuilder::PrepareOutputCcdb(DataAllocator& output)
void ITSMFTDeadMapBuilder::PrepareOutputCcdb(EndOfStreamContext* ec, std::string ccdburl = "")
{

// if ccdburl is specified, the object is sent to ccdb from this workflow

long tend = o2::ccdb::getCurrentTimestamp();

std::map<std::string, std::string> md = {
Expand All @@ -253,16 +257,33 @@ void ITSMFTDeadMapBuilder::PrepareOutputCcdb(DataAllocator& output)

info.setAdjustableEOV();

LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName()
<< " of size " << image->size() << "bytes, valid for "
<< info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();
if (ccdburl.empty()) {
nicolovalle marked this conversation as resolved.
Show resolved Hide resolved

if (mRunMFT) {
output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TimeDeadMap", 1}, *image.get());
output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TimeDeadMap", 1}, info);
} else {
output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TimeDeadMap", 0}, *image.get());
output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TimeDeadMap", 0}, info);
LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName()
<< "to ccdb-populator, of size " << image->size() << "bytes, valid for "
<< info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();

if (mRunMFT) {
ec->outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TimeDeadMap", 1}, *image.get());
ec->outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TimeDeadMap", 1}, info);
} else {
ec->outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TimeDeadMap", 0}, *image.get());
ec->outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TimeDeadMap", 0}, info);
}
}

else { // send from this workflow

LOG(info) << mSelfName << "sending object " << ccdburl << "/browse/" << info.getFileName()
<< " of size " << image->size() << "bytes, valid for "
<< info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();

o2::ccdb::CcdbApi mApi;
mApi.init(ccdburl);
mApi.storeAsBinaryFile(
&image->at(0), image->size(), info.getFileName(), info.getObjectType(),
info.getPath(), info.getMetaData(),
info.getStartValidityTimestamp(), info.getEndValidityTimestamp());
nicolovalle marked this conversation as resolved.
Show resolved Hide resolved
}

return;
Expand All @@ -273,11 +294,11 @@ void ITSMFTDeadMapBuilder::PrepareOutputCcdb(DataAllocator& output)
// tells that there will be no more input data
void ITSMFTDeadMapBuilder::endOfStream(EndOfStreamContext& ec)
{
if (!isEnded && !mRunStopRequested) {
LOG(info) << "endOfStream report:" << mSelfName;
if (!isEnded) {
LOG(info) << "endOfStream report: " << mSelfName;
finalizeOutput();
if (mMapObject.getEvolvingMapSize() > 0) {
PrepareOutputCcdb(ec.outputs());
PrepareOutputCcdb(&ec);
} else {
LOG(warning) << "Time-dependent dead map is empty and will not be forwarded as output";
}
Expand All @@ -291,12 +312,14 @@ void ITSMFTDeadMapBuilder::endOfStream(EndOfStreamContext& ec)
void ITSMFTDeadMapBuilder::stop()
{
if (!isEnded) {
LOG(info) << "stop() report:" << mSelfName;
LOG(info) << "stop() report: " << mSelfName;
finalizeOutput();
if (mDoLocalOutput) {
LOG(info) << "stop() not sending object as output. ccdb will not be populated.";
if (!mCCDBUrl.empty()) {
std::string detname = mRunMFT ? "MFT" : "ITS";
LOG(warning) << "endOfStream not processed. Sending output to ccdb from the " << detname << "deadmap builder workflow.";
PrepareOutputCcdb(nullptr, mCCDBUrl);
} else {
LOG(error) << "stop() not sending object as output. ccdb will not be populated.";
LOG(warning) << "endOfStream not processed. Nothing forwarded as output.";
}
isEnded = true;
}
Expand Down Expand Up @@ -342,6 +365,7 @@ DataProcessorSpec getITSMFTDeadMapBuilderSpec(std::string datasource, bool doMFT
Options{{"tf-sampling", VariantType::Int, 1000, {"Process every Nth TF. Selection according to first TF orbit."}},
{"tf-length", VariantType::Int, 32, {"Orbits per TF."}},
{"skip-static-map", VariantType::Bool, false, {"Do not fill static part of the map."}},
{"ccdb-url", VariantType::String, "", {"CCDB url. Ignored if endOfStream is processed."}},
{"outfile", VariantType::String, objectname_default, {"ROOT object file name."}},
{"local-output", VariantType::Bool, false, {"Save ROOT tree file locally."}},
{"output-dir", VariantType::String, "./", {"ROOT tree local output directory."}}}};
Expand Down
4 changes: 2 additions & 2 deletions prodtests/full-system-test/aggregator-workflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,11 @@ if [[ $AGGREGATOR_TASKS == BARREL_TF ]] || [[ $AGGREGATOR_TASKS == ALL ]]; then
fi
# ITS
if [[ $CALIB_ITS_DEADMAP_TIME == 1 ]]; then
add_W o2-itsmft-deadmap-builder-workflow "${CALIB_ITS_DEADMAP_TIME_OPT:-}"
add_W o2-itsmft-deadmap-builder-workflow "--ccdb-url $CCDB_POPULATOR_UPLOAD_PATH ${CALIB_ITS_DEADMAP_TIME_OPT:-}"
fi
# MFT
if [[ $CALIB_MFT_DEADMAP_TIME == 1 ]]; then
add_W o2-itsmft-deadmap-builder-workflow "--runmft ${CALIB_MFT_DEADMAP_TIME_OPT:---skip-static-map}"
add_W o2-itsmft-deadmap-builder-workflow "--runmft --ccdb-url $CCDB_POPULATOR_UPLOAD_PATH ${CALIB_MFT_DEADMAP_TIME_OPT:---skip-static-map}"
fi
# TOF
if [[ $CALIB_TOF_LHCPHASE == 1 ]] || [[ $CALIB_TOF_CHANNELOFFSETS == 1 ]]; then
Expand Down
Loading