Optional detailed report on the BasicCCDBManager cache (#12430)

* Add to the headers the ccdb fileSize entry

* Optional detailed report on BasicCCDBManager cache

* Add gettet for the total fetched size

---------

Co-authored-by: shahoian <ruben.shahoyan@cern.ch>
(cherry picked from commit 3301b3f)

Author: shahor02

CCDB/include/CCDB/BasicCCDBManager.h

@@ -48,6 +48,8 @@ class CCDBManagerInstance
     std::string uuid;
     long startvalidity = 0;
     long endvalidity = -1;
+    size_t minSize = -1ULL;
+    size_t maxSize = 0;
     int queries = 0;
     int fetches = 0;
     int failures = 0;
@@ -176,7 +178,9 @@ class CCDBManagerInstance
 
   std::string getSummaryString() const;
 
-  void endOfStream();
+  size_t getFetchedSize() const { return mFetchedSize; }
+
+  void report(bool longrep = false);
 
  private:
   // method to print (fatal) error
@@ -190,10 +194,11 @@ class CCDBManagerInstance
   bool mCanDefault = false;                             // whether default is ok --> useful for testing purposes done standalone/isolation
   bool mCachingEnabled = true;                          // whether caching is enabled
   bool mCheckObjValidityEnabled = false;                // wether the validity of cached object is checked before proceeding to a CCDB API query
+  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
   long mCreatedNotAfter = 0;                            // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header
   long mCreatedNotBefore = 0;                           // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header
   long mTimerMS = 0;                                    // timer for queries
-  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
+  size_t mFetchedSize = 0;                              // total fetched size
   int mQueries = 0;                                     // total number of object queries
   int mFetches = 0;                                     // total number of succesful fetches from CCDB
   int mFailures = 0;                                    // total number of failed fetches
@@ -218,11 +223,16 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       mFailures++;
     } else {
       mFetches++;
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+      }
     }
   } else {
     auto& cached = mCache[path];
+    cached.queries++;
     if (mCheckObjValidityEnabled && cached.isValid(timestamp)) {
-      cached.queries++;
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
@@ -254,6 +264,13 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       } catch (std::exception const& e) {
         reportFatal("Failed to read validity from CCDB response (Valid-From :  " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")"));
       }
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+        cached.minSize = std::min(s, cached.minSize);
+        cached.maxSize = std::max(s, cached.minSize);
+      }
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;
       cached.clear(); // in case of any error clear cache for this object

CCDB/include/CCDB/CcdbApi.h

@@ -542,7 +542,7 @@ class CcdbApi //: public DatabaseInterface
    * @param tcl The TClass object describing the serialized type
    * @return raw pointer to created object
    */
-  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo) const;
+  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map<string, string>* headers) const;
 
   // initialize the TGrid (Alien connection)
   bool initTGrid() const;

CCDB/src/BasicCCDBManager.cxx

@@ -54,7 +54,7 @@ std::pair<int64_t, int64_t> CCDBManagerInstance::getRunDuration(int runnumber, b
 
 std::string CCDBManagerInstance::getSummaryString() const
 {
-  std::string res = fmt::format("{} queries", mQueries);
+  std::string res = fmt::format("{} queries, {} bytes", mQueries, fmt::group_digits(mFetchedSize));
   if (mCachingEnabled) {
     res += fmt::format(" for {} objects", mCache.size());
   }
@@ -72,9 +72,15 @@ std::string CCDBManagerInstance::getSummaryString() const
   return res;
 }
 
-void CCDBManagerInstance::endOfStream()
+void CCDBManagerInstance::report(bool longrep)
 {
   LOG(info) << "CCDBManager summary: " << getSummaryString();
+  if (longrep && mCachingEnabled) {
+    LOGP(info, "CCDB cache miss/hit/failures");
+    for (const auto& obj : mCache) {
+      LOGP(info, "  {}: {}/{}/{} ({}-{} bytes)", obj.first, obj.second.fetches, obj.second.queries - obj.second.fetches - obj.second.failures, obj.second.failures, obj.second.minSize, obj.second.maxSize);
+    }
+  }
 }
 
 } // namespace ccdb

CCDB/src/CCDBDownloader.cxx

@@ -529,6 +529,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
             (*requestData->headers)["Error"] = "An error occurred during retrieval";
           }
           LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode);
+        } else {
+          if (requestData->headers && requestData->headers->find("fileSize") == requestData->headers->end()) {
+            (*requestData->headers)["fileSize"] = fmt::format("{}", requestData->hoPair.object ? requestData->hoPair.object->size() : 0);
+          }
         }
         --(*performData->requestsLeft);
         delete requestData;

CCDB/src/CcdbApi.cxx

@@ -824,6 +824,9 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info
     if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching
       (*headers)["ETag"] = filename;
     }
+    if (headers->find("fileSize") == headers->end()) {
+      (*headers)["fileSize"] = fmt::format("{}", f.GetEND());
+    }
   }
   return extractFromTFile(f, tcl);
 }
@@ -849,7 +852,7 @@ bool CcdbApi::initTGrid() const
   return mAlienInstance != nullptr;
 }
 
-void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo) const
+void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map<string, string>* headers) const
 {
   if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) {
     return nullptr;
@@ -859,6 +862,9 @@ void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info
   if (memfile) {
     auto cl = tinfo2TClass(tinfo);
     auto content = extractFromTFile(*memfile, cl);
+    if (headers && headers->find("fileSize") == headers->end()) {
+      (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND());
+    }
     delete memfile;
     return content;
   }
@@ -894,7 +900,7 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con
 
   // let's see first of all if the url is something specific that curl cannot handle
   if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) {
-    return downloadFilesystemContent(url, tinfo);
+    return downloadFilesystemContent(url, tinfo, headers);
   }
   // add other final cases here
   // example root://
@@ -925,6 +931,9 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con
     if (200 <= response_code && response_code < 300) {
       // good response and the content is directly provided and should have been dumped into "chunk"
       content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo);
+      if (headers && headers->find("fileSize") == headers->end()) {
+        (*headers)["fileSize"] = fmt::format("{}", chunk.size);
+      }
     } else if (response_code == 304) {
       // this means the object exist but I am not serving
       // it since it's already in your possession
@@ -1765,6 +1774,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& p
     if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching
       (*localHeaders)["ETag"] = path;
     }
+    if (localHeaders->find("fileSize") == localHeaders->end()) {
+      (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND());
+    }
   }
   return;
 }