From dd097b175f164d93da7765e827bcdd6fa0a006b6 Mon Sep 17 00:00:00 2001
From: Tor Egge <Tor.Egge@online.no>
Date: Fri, 26 Apr 2024 12:33:32 +0200
Subject: Expose imported attributes in metrics.

---
 .../proton/server/documentdb_metrics_updater.cpp   | 16 ++++++
 .../proton/server/fast_access_doc_subdb.cpp        | 66 +++++++++++++---------
 2 files changed, 55 insertions(+), 27 deletions(-)

(limited to 'searchcore/src')

diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
index dd66c7ceb46..3c7d197d5e1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
@@ -9,19 +9,23 @@
 #include <vespa/searchcommon/attribute/status.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
 #include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/docsummary/isummarymanager.h>
 #include <vespa/searchcore/proton/matching/matching_stats.h>
 #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
 #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h>
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/vespalib/stllike/cache_stats.h>
 #include <vespa/searchlib/util/searchable_stats.h>
 #include <vespa/vespalib/util/memoryusage.h>
+#include <vespa/vespalib/util/size_literals.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.documentdb_metrics_updater");
 
 using search::LidUsageStats;
+using search::attribute::ImportedAttributeVector;
 using vespalib::CacheStats;
 using vespalib::MemoryUsage;
 
@@ -141,6 +145,18 @@ fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics,
                     fillTempAttributeMetrics(*subMetrics, attr->getName(), memoryUsage, bitVectors);
                 }
             }
+            auto imported = attrMgr->getImportedAttributes();
+            if (imported != nullptr) {
+                std::vector<std::shared_ptr<ImportedAttributeVector>> i_list;
+                imported->getAll(i_list);
+                for (const auto& attr : i_list) {
+                    auto memory_usage = attr->get_memory_usage();
+                    fillTempAttributeMetrics(totalMetrics,  attr->getName(), memory_usage, 0);
+                    if (subMetrics != nullptr) {
+                        fillTempAttributeMetrics(*subMetrics,  attr->getName(), memory_usage, 0);
+                    }
+                }
+            }
         }
     }
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index a2d68ad8920..4972cc790c5 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -12,9 +12,11 @@
 #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h>
 #include <vespa/searchcore/proton/attribute/attribute_writer.h>
 #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/common/alloc_config.h>
 #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
 #include <vespa/searchcore/proton/reprocessing/reprocess_documents_task.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/vespalib/util/destructor_callbacks.h>
 
 #include <vespa/log/log.h>
@@ -23,6 +25,7 @@ LOG_SETUP(".proton.server.fast_access_doc_subdb");
 using search::AttributeGuard;
 using search::AttributeVector;
 using search::SerialNum;
+using search::attribute::ImportedAttributeVector;
 using search::index::Schema;
 using proton::initializer::InitializerTask;
 using searchcorespi::IFlushTarget;
@@ -85,16 +88,38 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co
                                                          attrMgrResult);
 }
 
+namespace {
+
+vespalib::hash_set<vespalib::string>
+get_attribute_names(const proton::IAttributeManager& mgr)
+{
+    vespalib::hash_set<vespalib::string> both;
+    std::vector<AttributeGuard> list;
+    mgr.getAttributeListAll(list);
+    for (const auto& attr : list) {
+        both.insert(attr->getName());
+    }
+    auto imported = mgr.getImportedAttributes();
+    if (imported != nullptr) {
+        std::vector<std::shared_ptr<ImportedAttributeVector>> i_list;
+        imported->getAll(i_list);
+        for (const auto& attr : i_list) {
+            both.insert(attr->getName());
+        }
+    }
+    return both;
+}
+
+}
+
 void
 FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult)
 {
     if (_addMetrics) {
         // register attribute metrics
-        std::vector<AttributeGuard> list;
-        attrMgrResult->getAttributeListAll(list);
+        auto list = get_attribute_names(*attrMgrResult);
         for (const auto &attr : list) {
-            const AttributeVector &v = *attr;
-            _metricsWireService.addAttribute(_subAttributeMetrics, v.getName());
+            _metricsWireService.addAttribute(_subAttributeMetrics, attr);
         }
     }
     _initAttrMgr = attrMgrResult;
@@ -141,33 +166,20 @@ void
 FastAccessDocSubDB::reconfigureAttributeMetrics(const proton::IAttributeManager &newMgr,
                                                 const proton::IAttributeManager &oldMgr)
 {
-    std::set<vespalib::string> toAdd;
-    std::set<vespalib::string> toRemove;
-    std::vector<AttributeGuard> newList;
-    std::vector<AttributeGuard> oldList;
-    newMgr.getAttributeList(newList);
-    oldMgr.getAttributeList(oldList);
-    for (const auto &newAttr : newList) {
-        if (std::find_if(oldList.begin(),
-                         oldList.end(),
-                         AttributeGuardComp(newAttr->getName())) ==
-            oldList.end()) {
-            toAdd.insert(newAttr->getName());
-        }
-    }
-    for (const auto &oldAttr : oldList) {
-        if (std::find_if(newList.begin(),
-                         newList.end(),
-                         AttributeGuardComp(oldAttr->getName())) ==
-            newList.end()) {
-            toRemove.insert(oldAttr->getName());
+    auto old_list = get_attribute_names(oldMgr);
+    auto new_list = get_attribute_names(newMgr);
+
+    for (const auto &attrName : new_list) {
+        if (old_list.contains(attrName)) {
+            continue;
         }
-    }
-    for (const auto &attrName : toAdd) {
         LOG(debug, "reconfigureAttributeMetrics(): addAttribute='%s'", attrName.c_str());
         _metricsWireService.addAttribute(_subAttributeMetrics, attrName);
     }
-    for (const auto &attrName : toRemove) {
+    for (const auto &attrName : old_list) {
+        if (new_list.contains(attrName)) {
+            continue;
+        }
         LOG(debug, "reconfigureAttributeMetrics(): removeAttribute='%s'", attrName.c_str());
         _metricsWireService.removeAttribute(_subAttributeMetrics, attrName);
     }
-- 
cgit v1.2.3