From 3a9fe55595d5338d635703b7c8161099d765a432 Mon Sep 17 00:00:00 2001 From: shahoian Date: Sun, 10 Dec 2023 02:19:08 +0100 Subject: [PATCH 1/3] Add to the headers the ccdb fileSize entry --- CCDB/include/CCDB/CcdbApi.h | 2 +- CCDB/src/CCDBDownloader.cxx | 4 ++++ CCDB/src/CcdbApi.cxx | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index a23b1a80130af..d54571416f8e2 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -542,7 +542,7 @@ class CcdbApi //: public DatabaseInterface * @param tcl The TClass object describing the serialized type * @return raw pointer to created object */ - void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo) const; + void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map* headers) const; // initialize the TGrid (Alien connection) bool initTGrid() const; diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx index 0a4b59fb9885e..bd2bf22d0add9 100644 --- a/CCDB/src/CCDBDownloader.cxx +++ b/CCDB/src/CCDBDownloader.cxx @@ -529,6 +529,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode) (*requestData->headers)["Error"] = "An error occurred during retrieval"; } LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode); + } else { + if (requestData->headers && requestData->headers->find("fileSize") == requestData->headers->end()) { + (*requestData->headers)["fileSize"] = fmt::format("{}", requestData->hoPair.object ? requestData->hoPair.object->size() : 0); + } } --(*performData->requestsLeft); delete requestData; diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 17252c30f6aab..0596d8798d895 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -832,6 +832,9 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching (*headers)["ETag"] = filename; } + if (headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", f.GetEND()); + } } return extractFromTFile(f, tcl); } @@ -857,7 +860,7 @@ bool CcdbApi::initTGrid() const return mAlienInstance != nullptr; } -void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo) const +void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map* headers) const { if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) { return nullptr; @@ -867,6 +870,9 @@ void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info if (memfile) { auto cl = tinfo2TClass(tinfo); auto content = extractFromTFile(*memfile, cl); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND()); + } delete memfile; return content; } @@ -902,7 +908,7 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con // let's see first of all if the url is something specific that curl cannot handle if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) { - return downloadFilesystemContent(url, tinfo); + return downloadFilesystemContent(url, tinfo, headers); } // add other final cases here // example root:// @@ -933,6 +939,9 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con if (200 <= response_code && response_code < 300) { // good response and the content is directly provided and should have been dumped into "chunk" content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", chunk.size); + } } else if (response_code == 304) { // this means the object exist but I am not serving // it since it's already in your possession @@ -1773,6 +1782,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& p if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching (*localHeaders)["ETag"] = path; } + if (localHeaders->find("fileSize") == localHeaders->end()) { + (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND()); + } } return; } From a5c0c5dee190be1daa7b96aff5f1aee33624d4bd Mon Sep 17 00:00:00 2001 From: shahoian Date: Sun, 10 Dec 2023 02:19:59 +0100 Subject: [PATCH 2/3] Optional detailed report on BasicCCDBManager cache --- CCDB/include/CCDB/BasicCCDBManager.h | 12 ++++++++++-- CCDB/src/BasicCCDBManager.cxx | 8 +++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index af1c0831cfcba..41738e1b9a02c 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -48,6 +48,8 @@ class CCDBManagerInstance std::string uuid; long startvalidity = 0; long endvalidity = -1; + size_t minSize = -1ULL; + size_t maxSize = 0; int queries = 0; int fetches = 0; int failures = 0; @@ -176,7 +178,7 @@ class CCDBManagerInstance std::string getSummaryString() const; - void endOfStream(); + void report(bool longrep = false); private: // method to print (fatal) error @@ -221,8 +223,8 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) } } else { auto& cached = mCache[path]; + cached.queries++; if (mCheckObjValidityEnabled && cached.isValid(timestamp)) { - cached.queries++; return reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); } ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, @@ -244,6 +246,12 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) } catch (std::exception const& e) { reportFatal("Failed to read validity from CCDB response (Valid-From : " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")")); } + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + cached.minSize = std::min(s, cached.minSize); + cached.maxSize = std::max(s, cached.minSize); + } } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set cached.failures++; cached.clear(); // in case of any error clear cache for this object diff --git a/CCDB/src/BasicCCDBManager.cxx b/CCDB/src/BasicCCDBManager.cxx index 148de1048b4d9..f55aa041261d2 100644 --- a/CCDB/src/BasicCCDBManager.cxx +++ b/CCDB/src/BasicCCDBManager.cxx @@ -72,9 +72,15 @@ std::string CCDBManagerInstance::getSummaryString() const return res; } -void CCDBManagerInstance::endOfStream() +void CCDBManagerInstance::report(bool longrep) { LOG(info) << "CCDBManager summary: " << getSummaryString(); + if (longrep && mCachingEnabled) { + LOGP(info, "CCDB cache miss/hit/failures"); + for (const auto& obj : mCache) { + LOGP(info, " {}: {}/{}/{} ({}-{} bytes)", obj.first, obj.second.fetches, obj.second.queries - obj.second.fetches - obj.second.failures, obj.second.failures, obj.second.minSize, obj.second.maxSize); + } + } } } // namespace ccdb From ad12117f69516d91f895e915757e7ac6c5ee8427 Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 12 Dec 2023 13:20:38 +0100 Subject: [PATCH 3/3] Add gettet for the total fetched size --- CCDB/include/CCDB/BasicCCDBManager.h | 11 ++++++++++- CCDB/src/BasicCCDBManager.cxx | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index 41738e1b9a02c..7158397fb25e5 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -178,6 +178,8 @@ class CCDBManagerInstance std::string getSummaryString() const; + size_t getFetchedSize() const { return mFetchedSize; } + void report(bool longrep = false); private: @@ -192,10 +194,11 @@ class CCDBManagerInstance bool mCanDefault = false; // whether default is ok --> useful for testing purposes done standalone/isolation bool mCachingEnabled = true; // whether caching is enabled bool mCheckObjValidityEnabled = false; // wether the validity of cached object is checked before proceeding to a CCDB API query + bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) long mCreatedNotAfter = 0; // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header long mCreatedNotBefore = 0; // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header long mTimerMS = 0; // timer for queries - bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) + size_t mFetchedSize = 0; // total fetched size int mQueries = 0; // total number of object queries int mFetches = 0; // total number of succesful fetches from CCDB int mFailures = 0; // total number of failed fetches @@ -220,6 +223,11 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) mFailures++; } else { mFetches++; + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + mFetchedSize += s; + } } } else { auto& cached = mCache[path]; @@ -249,6 +257,7 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) auto sh = mHeaders.find("fileSize"); if (sh != mHeaders.end()) { size_t s = atol(sh->second.c_str()); + mFetchedSize += s; cached.minSize = std::min(s, cached.minSize); cached.maxSize = std::max(s, cached.minSize); } diff --git a/CCDB/src/BasicCCDBManager.cxx b/CCDB/src/BasicCCDBManager.cxx index f55aa041261d2..8f86dd8866f26 100644 --- a/CCDB/src/BasicCCDBManager.cxx +++ b/CCDB/src/BasicCCDBManager.cxx @@ -54,7 +54,7 @@ std::pair CCDBManagerInstance::getRunDuration(int runnumber, b std::string CCDBManagerInstance::getSummaryString() const { - std::string res = fmt::format("{} queries", mQueries); + std::string res = fmt::format("{} queries, {} bytes", mQueries, fmt::group_digits(mFetchedSize)); if (mCachingEnabled) { res += fmt::format(" for {} objects", mCache.size()); }