From 3301b3fccb29bd1dfc44492fa1f96a412390bd0e Mon Sep 17 00:00:00 2001 From: Ruben Shahoyan Date: Mon, 8 Jan 2024 16:02:40 +0100 Subject: [PATCH] Optional detailed report on the BasicCCDBManager cache (#12430) * Add to the headers the ccdb fileSize entry * Optional detailed report on BasicCCDBManager cache * Add gettet for the total fetched size --------- Co-authored-by: shahoian --- CCDB/include/CCDB/BasicCCDBManager.h | 23 ++++++++++++++++++++--- CCDB/include/CCDB/CcdbApi.h | 2 +- CCDB/src/BasicCCDBManager.cxx | 10 ++++++++-- CCDB/src/CCDBDownloader.cxx | 4 ++++ CCDB/src/CcdbApi.cxx | 16 ++++++++++++++-- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index bdfe9cdcbbd4a..80b327b8cc380 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -48,6 +48,8 @@ class CCDBManagerInstance std::string uuid; long startvalidity = 0; long endvalidity = -1; + size_t minSize = -1ULL; + size_t maxSize = 0; int queries = 0; int fetches = 0; int failures = 0; @@ -176,7 +178,9 @@ class CCDBManagerInstance std::string getSummaryString() const; - void endOfStream(); + size_t getFetchedSize() const { return mFetchedSize; } + + void report(bool longrep = false); private: // method to print (fatal) error @@ -190,10 +194,11 @@ class CCDBManagerInstance bool mCanDefault = false; // whether default is ok --> useful for testing purposes done standalone/isolation bool mCachingEnabled = true; // whether caching is enabled bool mCheckObjValidityEnabled = false; // wether the validity of cached object is checked before proceeding to a CCDB API query + bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) long mCreatedNotAfter = 0; // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header long mCreatedNotBefore = 0; // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header long mTimerMS = 0; // timer for queries - bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) + size_t mFetchedSize = 0; // total fetched size int mQueries = 0; // total number of object queries int mFetches = 0; // total number of succesful fetches from CCDB int mFailures = 0; // total number of failed fetches @@ -218,11 +223,16 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) mFailures++; } else { mFetches++; + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + mFetchedSize += s; + } } } else { auto& cached = mCache[path]; + cached.queries++; if (mCheckObjValidityEnabled && cached.isValid(timestamp)) { - cached.queries++; return reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); } ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, @@ -254,6 +264,13 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) } catch (std::exception const& e) { reportFatal("Failed to read validity from CCDB response (Valid-From : " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")")); } + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + mFetchedSize += s; + cached.minSize = std::min(s, cached.minSize); + cached.maxSize = std::max(s, cached.minSize); + } } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set cached.failures++; cached.clear(); // in case of any error clear cache for this object diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index a23b1a80130af..d54571416f8e2 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -542,7 +542,7 @@ class CcdbApi //: public DatabaseInterface * @param tcl The TClass object describing the serialized type * @return raw pointer to created object */ - void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo) const; + void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map* headers) const; // initialize the TGrid (Alien connection) bool initTGrid() const; diff --git a/CCDB/src/BasicCCDBManager.cxx b/CCDB/src/BasicCCDBManager.cxx index 148de1048b4d9..8f86dd8866f26 100644 --- a/CCDB/src/BasicCCDBManager.cxx +++ b/CCDB/src/BasicCCDBManager.cxx @@ -54,7 +54,7 @@ std::pair CCDBManagerInstance::getRunDuration(int runnumber, b std::string CCDBManagerInstance::getSummaryString() const { - std::string res = fmt::format("{} queries", mQueries); + std::string res = fmt::format("{} queries, {} bytes", mQueries, fmt::group_digits(mFetchedSize)); if (mCachingEnabled) { res += fmt::format(" for {} objects", mCache.size()); } @@ -72,9 +72,15 @@ std::string CCDBManagerInstance::getSummaryString() const return res; } -void CCDBManagerInstance::endOfStream() +void CCDBManagerInstance::report(bool longrep) { LOG(info) << "CCDBManager summary: " << getSummaryString(); + if (longrep && mCachingEnabled) { + LOGP(info, "CCDB cache miss/hit/failures"); + for (const auto& obj : mCache) { + LOGP(info, " {}: {}/{}/{} ({}-{} bytes)", obj.first, obj.second.fetches, obj.second.queries - obj.second.fetches - obj.second.failures, obj.second.failures, obj.second.minSize, obj.second.maxSize); + } + } } } // namespace ccdb diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx index 0a4b59fb9885e..bd2bf22d0add9 100644 --- a/CCDB/src/CCDBDownloader.cxx +++ b/CCDB/src/CCDBDownloader.cxx @@ -529,6 +529,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode) (*requestData->headers)["Error"] = "An error occurred during retrieval"; } LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode); + } else { + if (requestData->headers && requestData->headers->find("fileSize") == requestData->headers->end()) { + (*requestData->headers)["fileSize"] = fmt::format("{}", requestData->hoPair.object ? requestData->hoPair.object->size() : 0); + } } --(*performData->requestsLeft); delete requestData; diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index defd53712c95a..da997b85e32f9 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -832,6 +832,9 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching (*headers)["ETag"] = filename; } + if (headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", f.GetEND()); + } } return extractFromTFile(f, tcl); } @@ -857,7 +860,7 @@ bool CcdbApi::initTGrid() const return mAlienInstance != nullptr; } -void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo) const +void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map* headers) const { if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) { return nullptr; @@ -867,6 +870,9 @@ void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info if (memfile) { auto cl = tinfo2TClass(tinfo); auto content = extractFromTFile(*memfile, cl); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND()); + } delete memfile; return content; } @@ -902,7 +908,7 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con // let's see first of all if the url is something specific that curl cannot handle if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) { - return downloadFilesystemContent(url, tinfo); + return downloadFilesystemContent(url, tinfo, headers); } // add other final cases here // example root:// @@ -933,6 +939,9 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con if (200 <= response_code && response_code < 300) { // good response and the content is directly provided and should have been dumped into "chunk" content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", chunk.size); + } } else if (response_code == 304) { // this means the object exist but I am not serving // it since it's already in your possession @@ -1773,6 +1782,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& p if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching (*localHeaders)["ETag"] = path; } + if (localHeaders->find("fileSize") == localHeaders->end()) { + (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND()); + } } return; }