Scout attribute index config.

Ignore live config if attribute types don't match.

6 files changed, 121 insertions, 24 deletions

diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp
index f76743b9480..2ae28936476 100644
--- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp
@@ -98,11 +98,32 @@ assertFastSearchAndMoreAttribute(const AttributesConfig::Attribute &attribute,
     return true;
 }
 
+bool
+assertTensorAttribute(const AttributesConfig::Attribute &attribute,
+                      const vespalib::string &name, const vespalib::string &spec, int max_links_per_node)
+{
+    if (!EXPECT_EQUAL(attribute.name, name)) {
+        return false;
+    }
+    if (!EXPECT_EQUAL((int)attribute.datatype, (int)AttributesConfig::Attribute::Datatype::TENSOR)) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(attribute.tensortype, spec)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.index.hnsw.enabled)) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(attribute.index.hnsw.maxlinkspernode, max_links_per_node)) {
+        return false;
+    }
+    return true;
+}
 
 bool
 assertAttributes(const AttributesConfig::AttributeVector &attributes)
 {
-    if (!EXPECT_EQUAL(4u, attributes.size())) {
+    if (!EXPECT_EQUAL(6u, attributes.size())) {
         return false;
     }
     if (!assertDefaultAttribute(attributes[0], "a1")) {
@@ -117,6 +138,12 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes)
     if (!assertDefaultAttribute(attributes[3], "a4")) {
         return false;
     }
+    if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 16)) {
+        return false;
+    }                                                                         
+    if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) {
+        return false;
+    }                                                                         
     return true;
 }
 
@@ -124,7 +151,7 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes)
 bool
 assertLiveAttributes(const AttributesConfig::AttributeVector &attributes)
 {
-    if (!EXPECT_EQUAL(5u, attributes.size())) {
+    if (!EXPECT_EQUAL(7u, attributes.size())) {
         return false;
     }
     if (!assertFastSearchAttribute(attributes[0], "a0")) {
@@ -142,6 +169,12 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes)
     if (!assertFastSearchAttribute(attributes[4], "a4")) {
         return false;
     }
+    if (!assertTensorAttribute(attributes[5], "tensor1", "tensor(x[100])", 32)) {
+        return false;
+    }                                                                         
+    if (!assertTensorAttribute(attributes[6], "tensor2", "tensor(x[200])", 32)) {
+        return false;
+    }                                                                         
     return true;
 }
 
@@ -149,7 +182,7 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes)
 bool
 assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes)
 {
-    if (!EXPECT_EQUAL(4u, attributes.size())) {
+    if (!EXPECT_EQUAL(6u, attributes.size())) {
         return false;
     }
     if (!assertFastSearchAndMoreAttribute(attributes[0], "a1")) {
@@ -164,6 +197,12 @@ assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes)
     if (!assertDefaultAttribute(attributes[3], "a4")) {
         return false;
     }
+    if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 32)) {
+        return false;
+    }                                                                         
+    if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) {
+        return false;
+    }                                                                         
     return true;
 }
 
@@ -199,6 +238,17 @@ setupFastSearchAndMoreAttribute(const vespalib::string name)
     return attribute;
 }
 
+AttributesConfig::Attribute
+setupTensorAttribute(const vespalib::string &name, const vespalib::string &spec, int max_links_per_node)
+{
+    AttributesConfig::Attribute attribute;
+    attribute.name = name;
+    attribute.datatype = AttributesConfig::Attribute::Datatype::TENSOR;
+    attribute.tensortype = spec;
+    attribute.index.hnsw.enabled = true;
+    attribute.index.hnsw.maxlinkspernode = max_links_per_node;
+    return attribute;
+}
 
 void
 setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes)
@@ -207,6 +257,8 @@ setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes)
     attributes.push_back(setupDefaultAttribute("a2"));
     attributes.push_back(setupDefaultAttribute("a3"));
     attributes.push_back(setupDefaultAttribute("a4"));
+    attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 16));
+    attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[100])", 16));
 }
 
 
@@ -221,6 +273,8 @@ setupLiveAttributes(AttributesConfigBuilder::AttributeVector &attributes)
     attributes.back().collectiontype = AttributesConfig::Attribute::Collectiontype::ARRAY;
     attributes.push_back(setupFastSearchAttribute("a4"));
     attributes.back().createifnonexistent = true;
+    attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 32));
+    attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[200])", 32));
 }
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
index f7a6ffe7189..550d0a1c4c4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
@@ -13,6 +13,7 @@ vespa_add_library(searchcore_attribute STATIC
     attribute_manager_initializer.cpp
     attribute_populator.cpp
     attribute_spec.cpp
+    attribute_type_matcher.cpp
     attribute_usage_filter.cpp
     attribute_usage_sampler_context.cpp
     attribute_usage_sampler_functor.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp
new file mode 100644
index 00000000000..c99a6ae9e6e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp
@@ -0,0 +1,34 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attribute_type_matcher.h"
+#include <vespa/searchcommon/attribute/config.h>
+
+using search::attribute::BasicType;
+
+namespace proton
+{
+
+bool
+AttributeTypeMatcher::operator()(const search::attribute::Config &oldConfig, const search::attribute::Config &newConfig) const
+{
+    if ((oldConfig.basicType() != newConfig.basicType()) ||
+        (oldConfig.collectionType() != newConfig.collectionType())) {
+        return false;
+    }
+    if (newConfig.basicType().type() == BasicType::Type::TENSOR) {
+        if (oldConfig.tensorType() != newConfig.tensorType()) {
+            return false;
+        }
+    }
+    if (newConfig.basicType().type() == BasicType::Type::PREDICATE) {
+        using Params = search::attribute::PersistentPredicateParams;
+        const Params &oldParams = oldConfig.predicateParams();
+        const Params &newParams = newConfig.predicateParams();
+        if (!(oldParams == newParams)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h
new file mode 100644
index 00000000000..bbe9479083b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h
@@ -0,0 +1,22 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search::attribute { class Config; }
+
+namespace proton
+{
+
+/**
+ * Class to check if attribute types are compatible or not.
+ */
+class AttributeTypeMatcher
+{
+public:
+    AttributeTypeMatcher() = default;
+    ~AttributeTypeMatcher() = default;
+    bool operator()(const search::attribute::Config &oldConfig,
+                    const search::attribute::Config &newConfig) const;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 5ff40fa5360..214db2c2d13 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -4,6 +4,7 @@
 #include "attribute_directory.h"
 #include "attributedisklayout.h"
 #include "attributemanager.h"
+#include "attribute_type_matcher.h"
 #include "imported_attributes_context.h"
 #include "imported_attributes_repo.h"
 #include "sequential_attributes_initializer.h"
@@ -41,25 +42,8 @@ namespace {
 
 bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Config &newConfig) {
     if (av) {
-        const auto &oldConfig = av->getConfig();
-        if ((oldConfig.basicType() != newConfig.basicType()) ||
-            (oldConfig.collectionType() != newConfig.collectionType())) {
-            return false;
-        }
-        if (newConfig.basicType().type() == BasicType::TENSOR) {
-            if (oldConfig.tensorType() != newConfig.tensorType()) {
-                return false;
-            }
-        }
-        if (newConfig.basicType().type() == BasicType::PREDICATE) {
-            using Params = search::attribute::PersistentPredicateParams;
-            const Params &oldParams = oldConfig.predicateParams();
-            const Params &newParams = newConfig.predicateParams();
-            if (!(oldParams == newParams)) {
-                return false;
-            }
-        }
-        return true;
+        AttributeTypeMatcher matching_types;
+        return matching_types(av->getConfig(), newConfig);
     } else {
         return false;
     }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp
index 2f6cf9e6208..986571d07e1 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "attributesconfigscout.h"
+#include "attribute_type_matcher.h"
 #include <vespa/searchlib/attribute/configconverter.h>
 
 using search::attribute::ConfigConverter;
@@ -29,6 +30,7 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr,
     attr.huge = liveAttr.huge;
     // Note: Predicate attributes only handle changes for the dense-posting-list-threshold config.
     attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold;
+    attr.index = liveAttr.index;
 }
 
 
@@ -41,8 +43,8 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr)
         const auto &liveAttr = _live.attribute[it->second];
         search::attribute::Config liveCfg =
             ConfigConverter::convert(liveAttr);
-        if (cfg.basicType() == liveCfg.basicType() &&
-            cfg.collectionType() == liveCfg.collectionType()) {
+        AttributeTypeMatcher matching_types;
+        if (matching_types(cfg, liveCfg)) {
             adjust(attr, liveAttr);
         }
     }