Check tensor type and predicate params before allowing transfer of attribute.

Don't try to load attribute of wrong tensor type.

4 files changed, 103 insertions, 11 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 5ccab84b4ad..50d79b42910 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -61,6 +61,7 @@ using search::predicate::PredicateIndex;
 using search::predicate::PredicateTreeAnnotations;
 using vespa::config::search::AttributesConfig;
 using vespa::config::search::AttributesConfigBuilder;
+using vespalib::eval::ValueType;
 
 typedef search::attribute::Config AVConfig;
 typedef proton::AttributeCollectionSpec::Attribute AttrSpec;
@@ -755,19 +756,44 @@ TEST_F("require that imported attributes are exposed via attribute context toget
 
 TEST_F("require that attribute vector of wrong type is dropped", BaseFixture)
 {
+    AVConfig generic_tensor(BasicType::TENSOR);
+    generic_tensor.setTensorType(ValueType::tensor_type({}));
+    AVConfig dense_tensor(BasicType::TENSOR);
+    dense_tensor.setTensorType(ValueType::from_spec("tensor(x[10])"));
+    AVConfig predicate(BasicType::PREDICATE);
+    predicate.setArity(2);
+    AVConfig predicate2(BasicType::PREDICATE);
+    predicate2.setArity(4);
+
     auto am1(std::make_shared<proton::AttributeManager>
              (test_dir, "test.subdb", TuneFileAttributes(),
               f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo));
     am1->addAttribute("a1", INT32_SINGLE, 1);
     am1->addAttribute("a2", INT32_SINGLE, 2);
+    am1->addAttribute("a3", generic_tensor, 3);
+    am1->addAttribute("a4", generic_tensor, 4);
+    am1->addAttribute("a5", predicate, 5);
+    am1->addAttribute("a6", predicate, 6);
     AttrSpecList newSpec;
     newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
     newSpec.push_back(AttrSpec("a2", INT32_ARRAY));
+    newSpec.push_back(AttrSpec("a3", generic_tensor));
+    newSpec.push_back(AttrSpec("a4", dense_tensor));
+    newSpec.push_back(AttrSpec("a5", predicate));
+    newSpec.push_back(AttrSpec("a6", predicate2));
     SequentialAttributeManager am2(*am1, AttrMgrSpec(newSpec, 5, 20));
     TEST_DO(assertCreateSerialNum(*am1, "a1", 1));
     TEST_DO(assertCreateSerialNum(*am1, "a2", 2));
+    TEST_DO(assertCreateSerialNum(*am1, "a3", 3));
+    TEST_DO(assertCreateSerialNum(*am1, "a4", 4));
+    TEST_DO(assertCreateSerialNum(*am1, "a5", 5));
+    TEST_DO(assertCreateSerialNum(*am1, "a6", 6));
     TEST_DO(assertCreateSerialNum(am2.mgr, "a1", 1));
     TEST_DO(assertCreateSerialNum(am2.mgr, "a2", 20));
+    TEST_DO(assertCreateSerialNum(am2.mgr, "a3", 3));
+    TEST_DO(assertCreateSerialNum(am2.mgr, "a4", 20));
+    TEST_DO(assertCreateSerialNum(am2.mgr, "a5", 5));
+    TEST_DO(assertCreateSerialNum(am2.mgr, "a6", 20));
 }
 
 TEST_MAIN()
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
index 1b998646a25..e2a3d5d1f5b 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
@@ -12,6 +12,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.attribute.attribute_initializer");
 
+using search::attribute::BasicType;
 using search::attribute::Config;
 using search::AttributeVector;
 using search::IndexMetaInfo;
@@ -25,6 +26,31 @@ namespace {
 const vespalib::string dataTypeTag = "datatype";
 const vespalib::string collectionTypeTag = "collectiontype";
 const vespalib::string createSerialNumTag = "createSerialNum";
+const vespalib::string tensorTypeTag = "tensortype";
+
+vespalib::string
+extraType(const Config &cfg)
+{
+    if (cfg.basicType().type() == BasicType::TENSOR) {
+        return cfg.tensorType().to_spec();
+    }
+    if (cfg.basicType().type() == BasicType::PREDICATE) {
+        // TODO: Check for predicate params
+    }
+    return "";
+}
+
+vespalib::string
+extraType(const AttributeHeader &header)
+{
+    if (header._btString == "tensor") {
+        return header._ttString;
+    }
+    if (header._btString == "predicate") {
+        // TODO: Check for predicate params
+    }
+    return "";
+}
 
 uint64_t
 extractCreateSerialNum(const vespalib::GenericHeader &header)
@@ -39,12 +65,25 @@ extractCreateSerialNum(const vespalib::GenericHeader &header)
 bool
 extractHeaderTypeOK(const vespalib::GenericHeader &header, const Config &cfg)
 {
-    return header.hasTag(dataTypeTag) &&
-        header.hasTag(collectionTypeTag) &&
-        header.getTag(dataTypeTag).asString() ==
-        cfg.basicType().asString() &&
-        header.getTag(collectionTypeTag).asString() ==
-        cfg.collectionType().asString();
+    if (!header.hasTag(dataTypeTag) || !header.hasTag(collectionTypeTag)) {
+        return false;
+    }
+    if ((header.getTag(dataTypeTag).asString() != cfg.basicType().asString()) ||
+        (header.getTag(collectionTypeTag).asString() != cfg.collectionType().asString())) {
+        return false;
+    }
+    if (cfg.basicType().type() == BasicType::TENSOR) {
+        if (!header.hasTag(tensorTypeTag)) {
+            return false;
+        }
+        if (header.getTag(tensorTypeTag).asString() != cfg.tensorType().to_spec()) {
+            return false;
+        }
+    }
+    if (cfg.basicType().type() == BasicType::PREDICATE) {
+        // TODO: Check parameters if available.
+    }
+    return true;
 }
 
 AttributeHeader
@@ -64,6 +103,9 @@ extractHeader(const AttributeVector::SP &attr,
     if (datHeader.hasTag(collectionTypeTag)) {
         retval._ctString = datHeader.getTag(collectionTypeTag).asString();
     }
+    if (datHeader.hasTag(tensorTypeTag)) {
+        retval._ttString = datHeader.getTag(tensorTypeTag).asString();
+    }
     return retval;
 }
 
@@ -83,12 +125,16 @@ logAttributeWrongType(const AttributeVector::SP &attr,
                       const AttributeHeader &header)
 {
     const Config &cfg(attr->getConfig());
-    LOG(info, "Attribute vector '%s' is of wrong type (expected %s/%s, got %s/%s)",
+    vespalib::string extraCfgType = extraType(cfg);
+    vespalib::string extraHeaderType = extraType(header);
+    LOG(info, "Attribute vector '%s' is of wrong type (expected %s/%s/%s, got %s/%s/%s)",
             attr->getBaseFileName().c_str(),
             cfg.basicType().asString(),
             cfg.collectionType().asString(),
+            extraCfgType.c_str(),
             header._btString.c_str(),
-            header._ctString.c_str());
+            header._ctString.c_str(),
+            extraHeaderType.c_str());
 }
 
 }
@@ -97,7 +143,8 @@ AttributeInitializer::AttributeHeader::AttributeHeader()
     : _createSerialNum(0),
       _headerTypeOK(false),
       _btString("unknown"),
-      _ctString("unknown")
+      _ctString("unknown"),
+      _ttString("unknown")
 {
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h
index d396c663710..b4f5d91e1f7 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h
@@ -25,6 +25,7 @@ public:
         bool _headerTypeOK;
         vespalib::string _btString;
         vespalib::string _ctString;
+        vespalib::string _ttString;
         AttributeHeader();
         ~AttributeHeader();
     };
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 5a44d87861d..b160f4067f5 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -27,6 +27,7 @@ using search::attribute::IAttributeContext;
 using search::attribute::IAttributeVector;
 using search::common::FileHeaderContext;
 using search::index::Schema;
+using search::attribute::BasicType;
 
 namespace proton {
 
@@ -35,8 +36,25 @@ namespace {
 bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Config &newConfig) {
     if (av) {
         const auto &oldConfig = av->getConfig();
-        return ((oldConfig.basicType() == newConfig.basicType()) &&
-                (oldConfig.collectionType() == newConfig.collectionType()));
+        if ((oldConfig.basicType() != newConfig.basicType()) ||
+            (oldConfig.collectionType() != newConfig.collectionType())) {
+            return false;
+        }
+        if (newConfig.basicType().type() == BasicType::TENSOR) {
+            if (oldConfig.tensorType() != newConfig.tensorType()) {
+                return false;
+            }
+        }
+        if (newConfig.basicType().type() == BasicType::PREDICATE) {
+            if ((oldConfig.getMaxInternalBlobSize() != newConfig.getMaxInternalBlobSize()) ||
+                (oldConfig.arity() != newConfig.arity()) ||
+                (oldConfig.lower_bound() != newConfig.lower_bound()) ||
+                (oldConfig.upper_bound() != newConfig.upper_bound()) ||
+                (oldConfig.dense_posting_list_threshold() != newConfig.dense_posting_list_threshold())) {
+                return false;
+            }
+        }
+        return true;
     } else {
         return false;
     }