Merge pull request #8493 from vespa-engine/toregge/verify-that-tensor-types-in-document-type-repo-are-propagated

Verify that tensor types (detailedtype) in document type repo are propagated.

6 files changed, 47 insertions, 5 deletions

diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
index 1e8fd9ec470..b92d85bd0ce 100644
--- a/document/src/tests/repo/documenttyperepo_test.cpp
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -530,4 +530,23 @@ TEST("Reference fields are resolved to correct reference type") {
     EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType());
 }
 
+TEST("Tensor fields have tensor types") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name, 
+                     Struct(header_name),
+                     Struct(body_name).
+                     addTensorField("tensor1", "tensor(x[3])").
+                     addTensorField("tensor2", "tensor(y{})").
+                     addTensorField("tensor3", "tensor(x[3])"));
+    DocumentTypeRepo repo(builder.config());
+    auto *docType = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(docType != nullptr);
+    auto &tensorField1 = docType->getField("tensor1");
+    auto &tensorField2 = docType->getField("tensor2");
+    EXPECT_EQUAL("TensorDataType(tensor(x[3]))", tensorField1.getDataType().toString());
+    EXPECT_EQUAL("TensorDataType(tensor(y{}))", tensorField2.getDataType().toString());
+    auto &tensorField3 = docType->getField("tensor3");
+    EXPECT_TRUE(&tensorField1.getDataType() == &tensorField3.getDataType());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/vespa/document/base/testdocrepo.cpp b/document/src/vespa/document/base/testdocrepo.cpp
index 026d29ed3b0..5bbb06519ac 100644
--- a/document/src/vespa/document/base/testdocrepo.cpp
+++ b/document/src/vespa/document/base/testdocrepo.cpp
@@ -52,7 +52,7 @@ DocumenttypesConfig TestDocRepo::getDefaultConfig() {
                      .addField("content", DataType::T_STRING)
                      .addField("rawarray", Array(DataType::T_RAW))
                      .addField("structarray", structarray_id)
-                     .addField("tensor", DataType::T_TENSOR));
+                     .addTensorField("tensor", "tensor(x{},y{})"));
     builder.document(type2_id, "testdoctype2",
                      Struct("testdoctype2.header")
                      .addField("onlyinchild", DataType::T_INT),
diff --git a/document/src/vespa/document/datatype/tensor_data_type.cpp b/document/src/vespa/document/datatype/tensor_data_type.cpp
index df799509ab1..e261c7fef2b 100644
--- a/document/src/vespa/document/datatype/tensor_data_type.cpp
+++ b/document/src/vespa/document/datatype/tensor_data_type.cpp
@@ -5,12 +5,20 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <sstream>
 
+using vespalib::eval::ValueType;
+
 namespace document {
 
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(TensorDataType, DataType);
 
 TensorDataType::TensorDataType()
-    : PrimitiveDataType(DataType::T_TENSOR)
+    : TensorDataType(ValueType::error_type())
+{
+}
+
+TensorDataType::TensorDataType(ValueType tensorType)
+    : PrimitiveDataType(DataType::T_TENSOR),
+      _tensorType(std::move(tensorType))
 {
 }
 
@@ -30,13 +38,13 @@ void
 TensorDataType::print(std::ostream& out, bool verbose, const std::string& indent) const
 {
     (void) verbose; (void) indent;
-    out << "TensorDataType()";
+    out << "TensorDataType(" << _tensorType << ")";
 }
 
 std::unique_ptr<const TensorDataType>
-TensorDataType::fromSpec([[maybe_unused]] const vespalib::string &spec)
+TensorDataType::fromSpec(const vespalib::string &spec)
 {
-    return std::make_unique<const TensorDataType>();
+    return std::make_unique<const TensorDataType>(ValueType::from_spec(spec));
 }
 
 } // document
diff --git a/document/src/vespa/document/datatype/tensor_data_type.h b/document/src/vespa/document/datatype/tensor_data_type.h
index aafc72eabfb..dc98870fe28 100644
--- a/document/src/vespa/document/datatype/tensor_data_type.h
+++ b/document/src/vespa/document/datatype/tensor_data_type.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "primitivedatatype.h"
+#include <vespa/eval/eval/value_type.h>
 
 namespace document {
 
@@ -9,8 +10,10 @@ namespace document {
  * This class describes a tensor type.
  */
 class TensorDataType : public PrimitiveDataType {
+    vespalib::eval::ValueType _tensorType;
 public:
     TensorDataType();
+    TensorDataType(vespalib::eval::ValueType tensorType);
     
     std::unique_ptr<FieldValue> createFieldValue() const override;
     TensorDataType* clone() const override;
diff --git a/document/src/vespa/document/repo/configbuilder.cpp b/document/src/vespa/document/repo/configbuilder.cpp
index 42b37104e04..9610697b84f 100644
--- a/document/src/vespa/document/repo/configbuilder.cpp
+++ b/document/src/vespa/document/repo/configbuilder.cpp
@@ -28,6 +28,17 @@ void DatatypeConfig::addNestedType(const TypeOrId &t) {
     }
 }
 
+Struct &
+Struct::addTensorField(const vespalib::string &name, const vespalib::string &spec) {
+    sstruct.field.resize(sstruct.field.size() + 1);
+    auto &field = sstruct.field.back();
+    field.name = name;
+    field.id = createFieldId(name, DataType::T_TENSOR);
+    field.datatype = DataType::T_TENSOR;
+    field.detailedtype = spec;
+    return *this;
+}
+
 namespace {
 
 void addType(const DatatypeConfig &type,
diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h
index 856ffaeda08..5f6f5548ae2 100644
--- a/document/src/vespa/document/repo/configbuilder.h
+++ b/document/src/vespa/document/repo/configbuilder.h
@@ -58,6 +58,7 @@ struct Struct : DatatypeConfig {
         sstruct.field.back().datatype = data_type.id;
         return *this;
     }
+    Struct &addTensorField(const vespalib::string &name, const vespalib::string &spec);
     Struct &setId(int32_t i) { DatatypeConfig::setId(i); return *this; }
 };