Report max address space used in attribute vector components from content nodes (proton) to the cluster controller.

This is more generic than explicit address space values for enum store and multi value.
This is used in the cluster controller to determine whether to block external feed.

3 files changed, 39 insertions, 47 deletions

diff --git a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp
index 4144b87c61d..7e1cf89ccdf 100644
--- a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp
@@ -1,11 +1,12 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/persistence/spi/resource_usage_listener.h>
 #include <vespa/persistence/spi/resource_usage.h>
+#include <vespa/persistence/spi/resource_usage_listener.h>
 #include <vespa/searchcore/proton/attribute/attribute_usage_stats.h>
 #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h>
 #include <vespa/searchcore/proton/persistenceengine/resource_usage_tracker.h>
 #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h>
+#include <vespa/searchlib/attribute/address_space_components.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <vespa/vespalib/util/idestructorcallback.h>
 #include <atomic>
@@ -131,7 +132,7 @@ struct AttributeUsageStatsBuilder
     ~AttributeUsageStatsBuilder();
 
     AttributeUsageStatsBuilder& reset() { stats = AttributeUsageStats(); return *this; }
-    AttributeUsageStatsBuilder& merge(const NamedAttribute& named_attribute, size_t used_enum_store, size_t used_multivalue);
+    AttributeUsageStatsBuilder& merge(const NamedAttribute& named_attribute, size_t used_address_space);
 
     AttributeUsageStats build() { return stats; }
 
@@ -140,11 +141,11 @@ struct AttributeUsageStatsBuilder
 AttributeUsageStatsBuilder::~AttributeUsageStatsBuilder() = default;
 
 AttributeUsageStatsBuilder&
-AttributeUsageStatsBuilder::merge(const NamedAttribute& named_attribute, size_t used_enum_store, size_t used_multivalue)
+AttributeUsageStatsBuilder::merge(const NamedAttribute& named_attribute, size_t used_address_space)
 {
-    vespalib::AddressSpace enum_store_usage(used_enum_store, 0, usage_limit);
-    vespalib::AddressSpace multivalue_usage(used_multivalue, 0, usage_limit);
-    search::AddressSpaceUsage as_usage(enum_store_usage, multivalue_usage);
+    vespalib::AddressSpace address_space_usage(used_address_space, 0, usage_limit);
+    search::AddressSpaceUsage as_usage;
+    as_usage.set("comp", address_space_usage);
     stats.merge(as_usage, named_attribute.attribute, named_attribute.subdb);
     return *this;
 }
@@ -153,11 +154,10 @@ double rel_usage(size_t usage) noexcept {
     return (double) usage / (double) usage_limit;
 }
 
-ResourceUsage make_resource_usage(const vespalib::string& enum_store_name, size_t used_enum_store, const vespalib::string &multivalue_name, size_t used_multivalue)
+ResourceUsage make_resource_usage(const vespalib::string& attr_name, size_t used_address_space)
 {
-    AttributeResourceUsage enum_store_usage(rel_usage(used_enum_store), enum_store_name);
-    AttributeResourceUsage multivalue_usage(rel_usage(used_multivalue), multivalue_name);
-    return ResourceUsage(0.0, 0.0, enum_store_usage, multivalue_usage);
+    AttributeResourceUsage address_space_usage(rel_usage(used_address_space), attr_name);
+    return ResourceUsage(0.0, 0.0, address_space_usage);
 }
 
 }
@@ -170,24 +170,24 @@ TEST_F(ResourceUsageTrackerTest, aggregates_attribute_usage)
     auto aul2 = _tracker->make_attribute_usage_listener("doctype2");
     AttributeUsageStatsBuilder b1;
     AttributeUsageStatsBuilder b2;
-    b1.merge(ready_a1, 10, 20).merge(ready_a2, 5, 30);
-    b2.merge(ready_a1, 15, 15);
+    b1.merge(ready_a1, 10).merge(ready_a2, 5);
+    b2.merge(ready_a1, 15);
     aul1->notify_attribute_usage(b1.build());
     aul2->notify_attribute_usage(b2.build());
-    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1", 15, "doctype1.0.ready.a2", 30), get_usage());
-    b1.merge(notready_a1, 5, 31);
+    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1.comp", 15), get_usage());
+    b1.merge(notready_a1, 16);
     aul1->notify_attribute_usage(b1.build());
-    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1", 15, "doctype1.2.notready.a1", 31), get_usage());
-    b1.reset().merge(ready_a1, 10, 20).merge(ready_a2, 5, 30);
+    EXPECT_EQ(make_resource_usage("doctype1.2.notready.a1.comp", 16), get_usage());
+    b1.reset().merge(ready_a1, 10).merge(ready_a2, 5);
     aul1->notify_attribute_usage(b1.build());
-    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1", 15, "doctype1.0.ready.a2", 30), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1.comp", 15), get_usage());
     aul2.reset();
-    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1", 10, "doctype1.0.ready.a2", 30), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1.comp", 10), get_usage());
     aul1.reset();
-    EXPECT_EQ(make_resource_usage("", 0, "", 0), get_usage());
+    EXPECT_EQ(make_resource_usage("", 0), get_usage());
     aul2 = _tracker->make_attribute_usage_listener("doctype2");
     aul2->notify_attribute_usage(b2.build());
-    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1", 15, "doctype2.0.ready.a1", 15), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype2.0.ready.a1.comp", 15), get_usage());
 }
 
 TEST_F(ResourceUsageTrackerTest, can_skip_scan_when_aggregating_attributes)
@@ -198,16 +198,16 @@ TEST_F(ResourceUsageTrackerTest, can_skip_scan_when_aggregating_attributes)
     auto aul2 = _tracker->make_attribute_usage_listener("doctype2");
     AttributeUsageStatsBuilder b1;
     AttributeUsageStatsBuilder b2;
-    b1.merge(ready_a1, 20, 20).merge(ready_a2, 5, 30);
-    b2.merge(ready_a1, 15, 15);
+    b1.merge(ready_a1, 20).merge(ready_a2, 5);
+    b2.merge(ready_a1, 15);
     aul1->notify_attribute_usage(b1.build());
-    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1", 20, "doctype1.0.ready.a2", 30), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1.comp", 20), get_usage());
     EXPECT_EQ(2u, get_update_count());
     aul1->notify_attribute_usage(b1.build());
-    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1", 20, "doctype1.0.ready.a2", 30), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1.comp", 20), get_usage());
     EXPECT_EQ(2u, get_update_count()); // usage for doctype1 has not changed
     aul2->notify_attribute_usage(b2.build());
-    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1", 20, "doctype1.0.ready.a2", 30), get_usage());
+    EXPECT_EQ(make_resource_usage("doctype1.0.ready.a1.comp", 20), get_usage());
     EXPECT_EQ(2u, get_update_count()); // usage for doctype2 is less than usage for doctype1
     aul2.reset();
     aul1.reset();
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp
index 9791b55f7b7..9f1c6e66b5c 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp
@@ -73,8 +73,7 @@ ResourceUsageTracker::ResourceUsageTracker(IDiskMemUsageNotifier& disk_mem_usage
       _listener(nullptr),
       _disk_mem_usage_notifier(disk_mem_usage_notifier),
       _attribute_usage(),
-      _attribute_enum_store_max_document_type(),
-      _attribute_multivalue_max_document_type()
+      _attribute_address_space_max_document_type()
 {
     _disk_mem_usage_notifier.addDiskMemUsageListener(this);
 }
@@ -88,7 +87,7 @@ void
 ResourceUsageTracker::notifyDiskMemUsage(DiskMemUsageState state)
 {
     std::lock_guard guard(_lock);
-    _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_enum_store_usage(), _resource_usage.get_attribute_multivalue_usage());
+    _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_address_space_usage());
     if (_listener != nullptr) {
         _listener->update_resource_usage(_resource_usage);
     }
@@ -125,6 +124,7 @@ namespace {
 
 bool same_usage(const AddressSpaceUsageStats &lhs, const AddressSpaceUsageStats &rhs) {
     return ((lhs.getUsage().usage() == rhs.getUsage().usage()) &&
+            (lhs.get_component_name() == rhs.get_component_name()) &&
             (lhs.getAttributeName() == rhs.getAttributeName()) &&
             (lhs.getSubDbName() == rhs.getSubDbName()));
 }
@@ -140,18 +140,15 @@ ResourceUsageTracker::notify_attribute_usage(const vespalib::string &document_ty
 {
     std::lock_guard guard(_lock);
     auto& old_usage = _attribute_usage[document_type];
-    if (same_usage(old_usage.enumStoreUsage(), attribute_usage.enumStoreUsage()) && 
-        same_usage(old_usage.multiValueUsage(), attribute_usage.multiValueUsage())) {
+    if (same_usage(old_usage.max_address_space_usage(), attribute_usage.max_address_space_usage())) {
         return; // usage for document type has not changed
     }
     old_usage = attribute_usage;
-    double enum_store_max = attribute_usage.enumStoreUsage().getUsage().usage();
-    double multivalue_max = attribute_usage.multiValueUsage().getUsage().usage();
-    double old_enum_store_max = _resource_usage.get_attribute_enum_store_usage().get_usage();
-    double old_multivalue_max = _resource_usage.get_attribute_multivalue_usage().get_usage();
+    double address_space_max = attribute_usage.max_address_space_usage().getUsage().usage();
+    double old_address_space_max = _resource_usage.get_attribute_address_space_usage().get_usage();
 
-    if (can_skip_scan(enum_store_max, old_enum_store_max, document_type == _attribute_enum_store_max_document_type) &&
-        can_skip_scan(multivalue_max, old_multivalue_max, document_type == _attribute_multivalue_max_document_type)) {
+    if (can_skip_scan(address_space_max, old_address_space_max,
+                      document_type == _attribute_address_space_max_document_type)) {
         return; // usage for document type is less than or equal to usage for other document types
     }
     if (scan_attribute_usage(false, guard) && _listener != nullptr) {
@@ -168,7 +165,7 @@ class MaxAttributeUsage
     double                        _max_usage;
 
     vespalib::string get_name() const {
-        return *_document_type + "." + _max->getSubDbName() + "." + _max->getAttributeName();
+        return *_document_type + "." + _max->getSubDbName() + "." + _max->getAttributeName() + "." + _max->get_component_name();
     }
 
 public:
@@ -203,23 +200,19 @@ public:
 bool
 ResourceUsageTracker::scan_attribute_usage(bool force_changed, std::lock_guard<std::mutex>&)
 {
-    MaxAttributeUsage enum_store_max;
-    MaxAttributeUsage multivalue_max;
+    MaxAttributeUsage address_space_max;
     for (const auto& kv : _attribute_usage) {
-        enum_store_max.sample(kv.first, kv.second.enumStoreUsage());
-        multivalue_max.sample(kv.first, kv.second.multiValueUsage());
+        address_space_max.sample(kv.first, kv.second.max_address_space_usage());
     }
     ResourceUsage new_resource_usage(_resource_usage.get_disk_usage(),
                                      _resource_usage.get_memory_usage(),
-                                     enum_store_max.get_max_resource_usage(),
-                                     multivalue_max.get_max_resource_usage());
+                                     address_space_max.get_max_resource_usage());
 
     bool changed = (new_resource_usage != _resource_usage) ||
                    force_changed;
     if (changed) {
         _resource_usage = std::move(new_resource_usage);
-        _attribute_enum_store_max_document_type = enum_store_max.get_document_type();
-        _attribute_multivalue_max_document_type = multivalue_max.get_document_type();
+        _attribute_address_space_max_document_type = address_space_max.get_document_type();
     }
     return changed;
 }
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.h
index 879e60aeeac..40d8598a8da 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.h
@@ -31,8 +31,7 @@ class ResourceUsageTracker : public std::enable_shared_from_this<ResourceUsageTr
     storage::spi::IResourceUsageListener* _listener;
     IDiskMemUsageNotifier&      _disk_mem_usage_notifier;
     vespalib::hash_map<vespalib::string, AttributeUsageStats> _attribute_usage;
-    vespalib::string            _attribute_enum_store_max_document_type;
-    vespalib::string            _attribute_multivalue_max_document_type;
+    vespalib::string            _attribute_address_space_max_document_type;
     void remove_listener();
     void remove_document_type(const vespalib::string &document_type);
     void notify_attribute_usage(const vespalib::string &document_type, const AttributeUsageStats &attribute_usage);