Use max address space used in attribute vector components to determine whether to block put and update operations in proton.

10 files changed, 45 insertions, 111 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
index a1c7b0a152d..18b9962003c 100644
--- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
@@ -95,65 +95,36 @@ TEST_F("Check that default filter allows write", Fixture)
 
 TEST_F("Check that enum store limit can be reached", Fixture)
 {
-    f.filter.setConfig(Fixture::Config(0.8, 1.0));
+    f.filter.setConfig(Fixture::Config(0.8));
     MyAttributeStats stats;
     stats.triggerEnumStoreLimit();
     f.setAttributeStats(stats);
-    f.testWrite("enumStoreLimitReached: { "
+    f.testWrite("addressSpaceLimitReached: { "
                 "action: \""
                 "add more content nodes"
                 "\", "
                 "reason: \""
-                "enum store address space used (0.9375) > limit (0.8)"
+                "max address space in attribute vector components used (0.9375) > limit (0.8)"
                 "\", "
-                "enumStore: { used: 32212254720, dead: 0, limit: 34359738368}, "
-                "attributeName: \"enumeratedName\", subdb: \"ready\"}");
+                "addressSpace: { used: 32212254720, dead: 0, limit: 34359738368}, "
+                "attributeName: \"enumeratedName\", componentName: \"enum-store\", subdb: \"ready\"}");
 }
 
 TEST_F("Check that multivalue limit can be reached", Fixture)
 {
-    f.filter.setConfig(Fixture::Config(1.0, 0.8));
+    f.filter.setConfig(Fixture::Config(0.8));
     MyAttributeStats stats;
     stats.triggerMultiValueLimit();
     f.setAttributeStats(stats);
-    f.testWrite("multiValueLimitReached: { "
+    f.testWrite("addressSpaceLimitReached: { "
                 "action: \""
                 "add more content nodes"
                 "\", "
                 "reason: \""
-                "multiValue address space used (0.992188) > limit (0.8)"
+                "max address space in attribute vector components used (0.992188) > limit (0.8)"
                 "\", "
-                "multiValue: { used: 133169152, dead: 0, limit: 134217728}, "
-                "attributeName: \"multiValueName\", subdb: \"ready\"}");
-}
-
-TEST_F("Check that both enumstore limit and multivalue limit can be reached",
-       Fixture)
-{
-    f.filter.setConfig(Fixture::Config(0.8, 0.8));
-    MyAttributeStats stats;
-    stats.triggerEnumStoreLimit();
-    stats.triggerMultiValueLimit();
-    f.setAttributeStats(stats);
-    f.testWrite("enumStoreLimitReached: { "
-                "action: \""
-                "add more content nodes"
-                "\", "
-                "reason: \""
-                "enum store address space used (0.9375) > limit (0.8)"
-                "\", "
-                "enumStore: { used: 32212254720, dead: 0, limit: 34359738368}, "
-                "attributeName: \"enumeratedName\", subdb: \"ready\"}"
-                ", "
-                "multiValueLimitReached: { "
-                "action: \""
-                "add more content nodes"
-                "\", "
-                "reason: \""
-                "multiValue address space used (0.992188) > limit (0.8)"
-                "\", "
-                "multiValue: { used: 133169152, dead: 0, limit: 134217728}, "
-                "attributeName: \"multiValueName\", subdb: \"ready\"}");
+                "addressSpace: { used: 133169152, dead: 0, limit: 134217728}, "
+                "attributeName: \"multiValueName\", componentName: \"multi-value\", subdb: \"ready\"}");
 }
 
 TEST_F("listener is updated when attribute stats change", Fixture)
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_stats/attribute_usage_stats_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_stats/attribute_usage_stats_test.cpp
index 025ae859993..b85d3b43f5b 100644
--- a/searchcore/src/tests/proton/attribute/attribute_usage_stats/attribute_usage_stats_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_stats/attribute_usage_stats_test.cpp
@@ -12,7 +12,7 @@ expect_max_usage(size_t used, const vespalib::string& attr_name,
                  const vespalib::string& comp_name, const vespalib::string& sub_name,
                  const AttributeUsageStats& stats)
 {
-    const auto& max = stats.max_usage();
+    const auto& max = stats.max_address_space_usage();
     EXPECT_EQ(used, max.getUsage().used());
     EXPECT_EQ(attr_name, max.getAttributeName());
     EXPECT_EQ(comp_name, max.get_component_name());
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index 3a51176b0d8..daaab2b823a 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -431,12 +431,20 @@ visit.ignoremaxbytes bool default=true
 ## When set to 0 (default) we use 1 separate thread per document database.
 initialize.threads int default = 0
 
+## Portion of max address space used in components in attribute vectors
+## before put and update operations in feed is blocked.
+writefilter.attribute.address_space_limit double default = 0.9
+
 ## Portion of enumstore address space that can be used before put and update
 ## portion of feed is blocked.
+## Deprecated -> Use address_space_limit
+## TODO: remove this when enum store is removed from AttributeUsageStats
 writefilter.attribute.enumstorelimit double default = 0.9
 
 ## Portion of attribute multivalue mapping address space that can be used
 ## before put and update portion of feed is blocked.
+## Deprecated -> Use address_space_limit
+## TODO: remove this when multi value is removed from AttributeUsageStats
 writefilter.attribute.multivaluelimit double default = 0.9
 
 ## Portion of physical memory that can be resident memory in anonymous mapping
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp
index e19e7976227..807ad020df0 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp
@@ -14,38 +14,24 @@ void makeAddressSpaceMessage(std::ostream &os,
     os << "{ used: " <<
         usage.getUsage().used() << ", dead: " <<
         usage.getUsage().dead() << ", limit: " <<
-        usage.getUsage().limit() << "}, attributeName: \"" <<
-        usage.getAttributeName() << "\", subdb: \"" <<
-        usage.getSubDbName() << "\"}";
+        usage.getUsage().limit() << "}, " <<
+        "attributeName: \"" << usage.getAttributeName() << "\", " <<
+        "componentName: \"" << usage.get_component_name() << "\", " <<
+        "subdb: \"" << usage.getSubDbName() << "\"}";
 }
 
-void makeEnumStoreMessage(std::ostream &os,
-                          double used, double limit,
-                          const AddressSpaceUsageStats &usage)
+void make_error_message(std::ostream &os,
+                        double used, double limit,
+                        const AddressSpaceUsageStats &usage)
 {
-    os << "enumStoreLimitReached: { "
+    os << "addressSpaceLimitReached: { "
         "action: \""
         "add more content nodes"
         "\", "
         "reason: \""
-        "enum store address space used (" << used << ") > "
+        "max address space in attribute vector components used (" << used << ") > "
         "limit (" << limit << ")"
-        "\", enumStore: ";
-    makeAddressSpaceMessage(os, usage);
-}
-
-void makeMultiValueMessage(std::ostream &os,
-                           double used, double limit,
-                           const AddressSpaceUsageStats &usage)
-{
-    os << "multiValueLimitReached: { "
-        "action: \""
-        "add more content nodes"
-        "\", "
-        "reason: \""
-        "multiValue address space used (" << used << ") > "
-        "limit (" << limit << ")"
-        "\", multiValue: ";
+        "\", addressSpace: ";
     makeAddressSpaceMessage(os, usage);
 }
 
@@ -57,20 +43,11 @@ AttributeUsageFilter::recalcState(const Guard &guard)
     (void) guard;
     bool hasMessage = false;
     std::ostringstream message;
-    const AddressSpaceUsageStats &enumStoreUsage = _attributeStats.enumStoreUsage();
-    double enumStoreUsed = enumStoreUsage.getUsage().usage();
-    if (enumStoreUsed > _config._enumStoreLimit) {
+    const auto &max_usage = _attributeStats.max_address_space_usage();
+    double used = max_usage.getUsage().usage();
+    if (used > _config._address_space_limit) {
         hasMessage = true;
-        makeEnumStoreMessage(message, enumStoreUsed, _config._enumStoreLimit, enumStoreUsage);
-    }
-    const AddressSpaceUsageStats &multiValueUsage = _attributeStats.multiValueUsage();
-    double multiValueUsed = multiValueUsage.getUsage().usage();
-    if (multiValueUsed > _config._multiValueLimit) {
-        if (hasMessage) {
-            message << ", ";
-        }
-        hasMessage = true;
-        makeMultiValueMessage(message, multiValueUsed, _config._multiValueLimit, multiValueUsage);
+        make_error_message(message, used, _config._address_space_limit, max_usage);
     }
     if (hasMessage) {
         _state = State(false, message.str());
@@ -126,20 +103,6 @@ AttributeUsageFilter::set_listener(std::unique_ptr<IAttributeUsageListener> list
     _listener = std::move(listener);
 }
 
-double
-AttributeUsageFilter::getEnumStoreUsedRatio() const
-{
-    Guard guard(_lock);
-    return _attributeStats.enumStoreUsage().getUsage().usage();
-}
-
-double
-AttributeUsageFilter::getMultiValueUsedRatio() const
-{
-    Guard guard(_lock);
-    return _attributeStats.multiValueUsage().getUsage().usage();
-}
-
 bool
 AttributeUsageFilter::acceptWriteOperation() const
 {
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h
index 45035b40864..4c25bdfd8ca 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h
@@ -42,8 +42,6 @@ public:
     AttributeUsageStats getAttributeUsageStats() const;
     void setConfig(Config config);
     void set_listener(std::unique_ptr<IAttributeUsageListener> listener);
-    double getEnumStoreUsedRatio() const;
-    double getMultiValueUsedRatio() const;
     bool acceptWriteOperation() const override;
     State getAcceptState() const override;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h
index 66f85191a0e..501ece82970 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h
@@ -4,7 +4,7 @@
 
 namespace proton {
 
-/*
+/**
  * Struct representing config for when to filter write operations
  * due to attribute resource usage (e.g. enum store and multivalue mapping).
  * If resource limit is reached then further writes are denied in
@@ -14,23 +14,18 @@ namespace proton {
  */
 struct AttributeUsageFilterConfig
 {
-    double _enumStoreLimit;
-    double _multiValueLimit;
+    double _address_space_limit;
 
     AttributeUsageFilterConfig() noexcept
-        : _enumStoreLimit(1.0),
-          _multiValueLimit(1.0)
-    { }
+        : _address_space_limit(1.0)
+    {}
 
-    AttributeUsageFilterConfig(double enumStoreLimit_in,
-                               double multiValueLimit_in) noexcept
-        : _enumStoreLimit(enumStoreLimit_in),
-          _multiValueLimit(multiValueLimit_in)
-    { }
+    AttributeUsageFilterConfig(double address_space_limit) noexcept
+        : _address_space_limit(address_space_limit)
+    {}
 
     bool operator==(const AttributeUsageFilterConfig &rhs) const noexcept {
-        return ((_enumStoreLimit == rhs._enumStoreLimit) &&
-                (_multiValueLimit == rhs._multiValueLimit));
+        return (_address_space_limit == rhs._address_space_limit);
     }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp
index a077c890d18..f0ab56562a6 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp
@@ -34,7 +34,7 @@ operator<<(std::ostream& out, const AttributeUsageStats& rhs)
 {
     out << "{enum_store=" << rhs.enumStoreUsage() <<
             ", multi_value=" << rhs.multiValueUsage() <<
-            ", max_usage=" << rhs.max_usage() << "}";
+            ", max_address_space_usage=" << rhs.max_address_space_usage() << "}";
     return out;
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h
index 53019c856ce..762cc324f89 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h
@@ -27,7 +27,7 @@ public:
 
     const AddressSpaceUsageStats& enumStoreUsage() const { return _enumStoreUsage; }
     const AddressSpaceUsageStats& multiValueUsage() const { return _multiValueUsage; }
-    const AddressSpaceUsageStats& max_usage() const { return _max_usage; }
+    const AddressSpaceUsageStats& max_address_space_usage() const { return _max_usage; }
 
     bool operator==(const AttributeUsageStats& rhs) const {
         return (_enumStoreUsage == rhs._enumStoreUsage) &&
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
index 66ecf955736..956e9ea198e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
@@ -306,7 +306,7 @@ DocumentDBMetricsUpdater::updateAttributeResourceUsageMetrics(DocumentDBTaggedMe
     bool feedBlocked = !_writeFilter.acceptWriteOperation();
     double enumStoreUsed = stats.enumStoreUsage().getUsage().usage();
     double multiValueUsed = stats.multiValueUsage().getUsage().usage();
-    double address_space_used = stats.max_usage().getUsage().usage();
+    double address_space_used = stats.max_address_space_usage().getUsage().usage();
     metrics.resourceUsage.enumStore.set(enumStoreUsed);
     metrics.resourceUsage.multiValue.set(multiValueUsed);
     metrics.resourceUsage.address_space.set(address_space_used);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index fb154081963..184bc339d98 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -151,8 +151,7 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig,
                     proton.lidspacecompaction.removeblockrate,
                     isDocumentTypeGlobal),
             AttributeUsageFilterConfig(
-                    proton.writefilter.attribute.enumstorelimit,
-                    proton.writefilter.attribute.multivaluelimit),
+                    proton.writefilter.attribute.addressSpaceLimit),
             vespalib::from_s(proton.writefilter.sampleinterval),
             BlockableMaintenanceJobConfig(
                     proton.maintenancejobs.resourcelimitfactor,