Revert "Revert "Revert "Revert "Bratseth/tensor type info in documents""""

12 files changed, 142 insertions, 103 deletions

diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
index 413d1581e58..02773c7dad0 100644
--- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -10,6 +10,7 @@ import com.yahoo.document.update.FieldUpdate;
 import com.yahoo.document.update.ValueUpdate;
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -43,6 +44,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
 
     private final String documentId = "doc:something:foooo";
     private final String tensorField = "tensorfield";
+    private final TensorType tensorType = new TensorType.Builder().mapped("x").build();
 
     private Document createDocument() {
         return new Document(docMan.getDocumentType("foobar"), new DocumentId(documentId));
@@ -60,7 +62,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
         DataType stringwset = DataType.getWeightedSet(DataType.STRING);
         docType.addField(new Field("strwset", stringwset));
 
-        docType.addField(new Field(tensorField, DataType.TENSOR));
+        docType.addField(new Field(tensorField, new TensorDataType(tensorType)));
         docMan.register(docType);
 
         docType2 = new DocumentType("otherdoctype");
@@ -625,7 +627,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
     private DocumentUpdate createTensorAssignUpdate() {
         DocumentUpdate result = new DocumentUpdate(docType, new DocumentId(documentId));
         result.addFieldUpdate(FieldUpdate.createAssign(docType.getField(tensorField),
-                createTensorFieldValue("{{x:0}:2.0}")));
+                              createTensorFieldValue("{{x:0}:2.0}")));
         return result;
     }
 
diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
index 6f841aac821..6f3abd85521 100644
--- a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
+++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
@@ -9,16 +9,17 @@ import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertThat;
 
 /**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.10
+ * @author Einar M R Rosenvinge
  */
 public class TemporaryDataTypeTestCase {
+
     @Test
     public void requireNulls() {
-        TemporaryDataType type = new TemporaryDataType(0);
+        TemporaryDataType type = new TemporaryDataType(0, "");
         assertThat(type.createFieldValue(new Object()), nullValue());
         assertThat(type.createFieldValue(), nullValue());
         assertThat(type.getValueClass(), nullValue());
         assertThat(type.isValueCompatible(new StringFieldValue("")), is(false));
     }
+
 }
diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
index 80386141968..c94c917d2ca 100644
--- a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
+++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
@@ -2,6 +2,7 @@
 package com.yahoo.document.datatypes;
 
 import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
 import org.junit.Test;
 
 import static org.junit.Assert.assertFalse;
@@ -13,8 +14,8 @@ import static org.junit.Assert.assertTrue;
  */
 public class TensorFieldValueTestCase {
 
-    private static TensorFieldValue createFieldValue(String tensor) {
-        return new TensorFieldValue(Tensor.from(tensor));
+    private static TensorFieldValue createFieldValue(String tensorString) {
+        return new TensorFieldValue(Tensor.from(tensorString));
     }
 
     @Test
@@ -23,20 +24,27 @@ public class TensorFieldValueTestCase {
     }
 
     @Test
+    public void requireThatDifferentTensorTypesWithEmptyValuesAreNotEqual() {
+        TensorFieldValue field1 = new TensorFieldValue(new TensorType.Builder().mapped("x").build());
+        TensorFieldValue field2 = new TensorFieldValue(new TensorType.Builder().indexed("y").build());
+        assertFalse(field1.equals(field2));
+    }
+
+    @Test
     public void requireThatDifferentTensorValuesAreNotEqual() {
-        TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}");
-        TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}");
-        assertFalse(lhs.equals(rhs));
-        assertFalse(lhs.equals(new TensorFieldValue()));
+        TensorFieldValue field1 = createFieldValue("{{x:0}:2.0}");
+        TensorFieldValue field2 = createFieldValue("{{x:0}:3.0}");
+        assertFalse(field1.equals(field2));
+        assertFalse(field1.equals(new TensorFieldValue(TensorType.empty)));
     }
 
     @Test
     public void requireThatSameTensorValueIsEqual() {
         Tensor tensor = Tensor.from("{{x:0}:2.0}");
-        TensorFieldValue lhs = new TensorFieldValue(tensor);
-        TensorFieldValue rhs = new TensorFieldValue(tensor);
-        assertTrue(lhs.equals(lhs));
-        assertTrue(lhs.equals(rhs));
-        assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}")));
+        TensorFieldValue field1 = new TensorFieldValue(tensor);
+        TensorFieldValue field2 = new TensorFieldValue(tensor);
+        assertTrue(field1.equals(field1));
+        assertTrue(field1.equals(field2));
+        assertTrue(field1.equals(createFieldValue("{{x:0}:2.0}")));
     }
 }
diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
index ffec7927ab3..a0f993fd2fc 100644
--- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
+++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
@@ -10,7 +10,9 @@ import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.PositionDataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.tensor.TensorType;
 import com.yahoo.text.Utf8;
 import org.junit.Test;
 
@@ -27,6 +29,7 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson;
  */
 public class DocumentUpdateJsonSerializerTest {
 
+    final static TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build();
     final static DocumentTypeManager types = new DocumentTypeManager();
     final static JsonFactory parserFactory = new JsonFactory();
     final static DocumentType docType = new DocumentType("doctype");
@@ -39,7 +42,7 @@ public class DocumentUpdateJsonSerializerTest {
         docType.addField(new Field("float_field", DataType.FLOAT));
         docType.addField(new Field("double_field", DataType.DOUBLE));
         docType.addField(new Field("byte_field", DataType.BYTE));
-        docType.addField(new Field("tensor_field", DataType.TENSOR));
+        docType.addField(new Field("tensor_field", new TensorDataType(tensorType)));
         docType.addField(new Field("predicate_field", DataType.PREDICATE));
         docType.addField(new Field("raw_field", DataType.RAW));
         docType.addField(new Field("int_array", new ArrayDataType(DataType.INT)));
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 6c46f743332..34c93580eb3 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -19,6 +19,7 @@ import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.PositionDataType;
 import com.yahoo.document.StructDataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.datatypes.Array;
 import com.yahoo.document.datatypes.FieldValue;
@@ -37,7 +38,10 @@ import com.yahoo.document.update.ClearValueUpdate;
 import com.yahoo.document.update.FieldUpdate;
 import com.yahoo.document.update.MapValueUpdate;
 import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.tensor.IndexedTensor;
+import com.yahoo.tensor.MappedTensor;
 import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
 import com.yahoo.text.Utf8;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.After;
@@ -66,11 +70,12 @@ import static org.junit.Assert.*;
 /**
  * Basic test of JSON streams to Vespa document instances.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class JsonReaderTestCase {
-    DocumentTypeManager types;
-    JsonFactory parserFactory;
+
+    private DocumentTypeManager types;
+    private JsonFactory parserFactory;
 
     @Rule
     public ExpectedException exception = ExpectedException.none();
@@ -133,7 +138,10 @@ public class JsonReaderTestCase {
         }
         {
             DocumentType x = new DocumentType("testtensor");
-            x.addField(new Field("tensorfield", DataType.TENSOR));
+            x.addField(new Field("mappedtensorfield", 
+                                 new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build())));
+            x.addField(new Field("indexedtensorfield", 
+                                 new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build())));
             types.registerDocumentType(x);
         }
         {
@@ -1022,83 +1030,88 @@ public class JsonReaderTestCase {
         Document doc = createPutWithoutTensor().getDocument();
         assertEquals("testtensor", doc.getId().getDocType());
         assertEquals("id:unittest:testtensor::0", doc.getId().toString());
-        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("mappedtensorfield"));
         assertNull(fieldValue);
     }
 
     @Test
     public void testParsingOfEmptyTensor() {
-        assertTensorField("{}", createPutWithTensor("{}"));
+        assertMappedTensorField("tensor(x{},y{}):{}", createPutWithMappedTensor("{}"));
     }
 
     @Test
     public void testParsingOfTensorWithEmptyDimensions() {
-        assertTensorField("{}",
-                createPutWithTensor("{ "
-                        + "  \"dimensions\": [] "
-                        + "}"));
+        assertMappedTensorField("tensor(x{},y{}):{}",
+                                createPutWithMappedTensor("{ "
+                                                    + "  \"dimensions\": [] "
+                                                    + "}"));
     }
 
     @Test
     public void testParsingOfTensorWithEmptyCells() {
-        assertTensorField("{}",
-                createPutWithTensor("{ "
-                        + "  \"cells\": [] "
-                        + "}"));
+        assertMappedTensorField("tensor(x{},y{}):{}",
+                                createPutWithMappedTensor("{ "
+                                                    + "  \"cells\": [] "
+                                                    + "}"));
     }
 
     @Test
-    public void testParsingOfTensorWithCells() {
-        assertTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
-                createPutWithTensor("{ "
-                        + "  \"cells\": [ "
-                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
-                        + "      \"value\": 2.0 }, "
-                        + "    { \"address\": { \"x\": \"c\", \"y\": \"b\" }, "
-                        + "      \"value\": 3.0 } "
-                        + "  ]"
-                        + "}"));
+    public void testParsingOfMappedTensorWithCells() {
+        Tensor tensor = assertMappedTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
+                                createPutWithMappedTensor("{ "
+                                                    + "  \"cells\": [ "
+                                                    + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                                                    + "      \"value\": 2.0 }, "
+                                                    + "    { \"address\": { \"x\": \"c\", \"y\": \"b\" }, "
+                                                    + "      \"value\": 3.0 } "
+                                                    + "  ]"
+                                                    + "}"));
+        assertTrue(tensor instanceof MappedTensor); // any functional instance is fine
     }
 
     @Test
-    public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
-        assertTensorField("{{x:a,y:b}:2.0}",
-                createPutWithTensor("{ "
-                        + "  \"cells\": [ "
-                        + "    { \"value\": 2.0, "
-                        + "      \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
-                        + "  ]"
-                        + "}"));
+    public void testParsingOfIndexedTensorWithCells() {
+        Tensor tensor = assertTensorField("{{x:0,y:0}:2.0,{x:1,y:0}:3.0}}",
+                           createPutWithTensor("{ "
+                                                + "  \"cells\": [ "
+                                                + "    { \"address\": { \"x\": \"0\", \"y\": \"0\" }, "
+                                                + "      \"value\": 2.0 }, "
+                                                + "    { \"address\": { \"x\": \"1\", \"y\": \"0\" }, "
+                                                + "      \"value\": 3.0 } "
+                                                + "  ]"
+                                                + "}", "indexedtensorfield"), "indexedtensorfield");
+        assertTrue(tensor instanceof IndexedTensor); // this matters for performance
     }
 
     @Test
-    public void testParsingOfTensorWithSingleCellWithoutAddress() {
-        assertTensorField("{{}:2.0}",
-                createPutWithTensor("{ "
-                        + "  \"cells\": [ "
-                        + "    { \"value\": 2.0 } "
-                        + "  ]"
-                        + "}"));
+    public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
+        assertMappedTensorField("{{x:a,y:b}:2.0}",
+                                createPutWithMappedTensor("{ "
+                                                    + "  \"cells\": [ "
+                                                    + "    { \"value\": 2.0, "
+                                                    + "      \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
+                                                    + "  ]"
+                                                    + "}"));
     }
 
     @Test
-    public void testParsingOfTensorWithSingleCellWithoutValue() {
-        assertTensorField("{{x:a}:0.0}",
-                createPutWithTensor("{ "
-                        + "  \"cells\": [ "
-                        + "    { \"address\": { \"x\": \"a\" } } "
-                        + "  ]"
-                        + "}"));
+    public void testAssignUpdateOfEmptyMappedTensor() {
+        assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithMappedTensor("{}"));
     }
 
     @Test
-    public void testAssignUpdateOfEmptyTensor() {
-        assertTensorAssignUpdate("{}", createAssignUpdateWithTensor("{}"));
+    public void testAssignUpdateOfEmptyIndexedTensor() {
+        try {
+            assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithTensor("{}", "indexedtensorfield"));
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("An indexed tensor must have a value", "Tensor of type tensor(x[],y[]) has no values", e.getMessage());
+        }
     }
 
     @Test
     public void testAssignUpdateOfNullTensor() {
-        ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithTensor(null)).getValueUpdate(0);
+        ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithMappedTensor(null)).getValueUpdate(0);
         assertTrue(clearUpdate != null);
         assertTrue(clearUpdate.getValue() == null);
     }
@@ -1106,7 +1119,7 @@ public class JsonReaderTestCase {
     @Test
     public void testAssignUpdateOfTensorWithCells() {
         assertTensorAssignUpdate("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
-                createAssignUpdateWithTensor("{ "
+                createAssignUpdateWithMappedTensor("{ "
                         + "  \"cells\": [ "
                         + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
                         + "      \"value\": 2.0 }, "
@@ -1194,10 +1207,13 @@ public class JsonReaderTestCase {
         return (DocumentPut) reader.next();
     }
 
-    private DocumentPut createPutWithTensor(String inputTensor) {
+    private DocumentPut createPutWithMappedTensor(String inputTensor) { 
+        return createPutWithTensor(inputTensor, "mappedtensorfield"); 
+    }
+    private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("["
-                        + "  { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": "
+                        + "  { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": "
                         + inputTensor
                         + "  }}"
                         + "]"));
@@ -1205,20 +1221,27 @@ public class JsonReaderTestCase {
         return (DocumentPut) reader.next();
     }
 
-    private DocumentUpdate createAssignUpdateWithTensor(String inputTensor) {
+    private DocumentUpdate createAssignUpdateWithMappedTensor(String inputTensor) {
+        return createAssignUpdateWithTensor(inputTensor, "mappedtensorfield");
+    }
+    private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) {
         InputStream rawDoc = new ByteArrayInputStream(
-                Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {"
+                Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": {"
                         + "\"assign\": " + (inputTensor != null ? inputTensor : "null") + " } } } ]"));
         JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
         return (DocumentUpdate) reader.next();
     }
 
-    private static void assertTensorField(String expectedTensor, DocumentPut put) {
+    private static Tensor assertMappedTensorField(String expectedTensor, DocumentPut put) {
+        return assertTensorField(expectedTensor, put, "mappedtensorfield");
+    }
+    private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) {
         final Document doc = put.getDocument();
         assertEquals("testtensor", doc.getId().getDocType());
         assertEquals(TENSOR_DOC_ID, doc.getId().toString());
-        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField(tensorFieldName));
         assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get());
+        return fieldValue.getTensor().get();
     }
 
     private static void assertTensorAssignUpdate(String expectedTensor, DocumentUpdate update) {
@@ -1230,7 +1253,7 @@ public class JsonReaderTestCase {
     }
 
     private static FieldUpdate getTensorField(DocumentUpdate update) {
-        FieldUpdate fieldUpdate = update.getFieldUpdate("tensorfield");
+        FieldUpdate fieldUpdate = update.getFieldUpdate("mappedtensorfield");
         assertEquals(1, fieldUpdate.size());
         return fieldUpdate;
     }
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index f29c208a113..57b3e088efc 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -19,8 +19,10 @@ import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.PositionDataType;
 import com.yahoo.document.StructDataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.tensor.TensorType;
 import com.yahoo.text.Utf8;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.After;
@@ -115,7 +117,8 @@ public class JsonWriterTestCase {
         }
         {
             DocumentType x = new DocumentType("testtensor");
-            x.addField(new Field("tensorfield", DataType.TENSOR));
+            TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build();
+            x.addField(new Field("tensorfield", new TensorDataType(tensorType)));
             types.registerDocumentType(x);
         }
     }
@@ -298,7 +301,7 @@ public class JsonWriterTestCase {
 
     @Test
     public void testWritingOfEmptyTensor() throws IOException {
-        assertTensorRoundTripEquality("{}","{ \"cells\": [{\"address\": {}, \"value\": 0.0}] }");
+        assertTensorRoundTripEquality("{}","{ \"cells\": [] }");
     }
 
     @Test
@@ -321,20 +324,12 @@ public class JsonWriterTestCase {
     }
 
     @Test
-    public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException {
-        assertTensorRoundTripEquality("{ "
-                + "  \"cells\": [ "
-                + "    { \"address\": {}, \"value\": 2.0 } "
-                + "  ]"
-                + "}");
-    }
-
-    @Test
     public void testWritingOfTensorFieldValueWithoutTensor() throws IOException {
-        DocumentType tensorType = types.getDocumentType("testtensor");
+        DocumentType documentTypeWithTensor = types.getDocumentType("testtensor");
         String docId = "id:unittest:testtensor::0";
-        Document doc = new Document(tensorType, docId);
-        doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue());
+        Document doc = new Document(documentTypeWithTensor, docId);
+        Field tensorField = documentTypeWithTensor.getField("tensorfield");
+        doc.setFieldValue(tensorField, new TensorFieldValue(((TensorDataType)tensorField.getDataType()).getTensorType()));
         assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc));
     }
 
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
index f3987085e32..7e3fabc30fb 100644
--- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
@@ -2,6 +2,7 @@
 package com.yahoo.document.serialization;
 
 import com.yahoo.document.Document;
+import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.io.GrowableByteBuffer;
 
 import java.io.IOException;
diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
index 22cb35ae937..ae61bb3cf6f 100644
--- a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
+++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
@@ -4,8 +4,10 @@ package com.yahoo.document.serialization;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -19,30 +21,31 @@ import static org.junit.Assert.assertEquals;
  */
 public class TensorFieldValueSerializationTestCase {
 
+    private final static TensorType tensorType = new TensorType.Builder().mapped("dimX").mapped("dimY").build();
     private final static String TENSOR_FIELD = "my_tensor";
     private final static String TENSOR_FILES = "src/test/resources/tensor/";
-    private final static TestDocumentFactory docFactory =
-            new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
+    private final static TestDocumentFactory docFactory = new TestDocumentFactory(createDocType(), 
+                                                                                  "id:test:my_type::foo");
 
     private static DocumentType createDocType() {
         DocumentType type = new DocumentType("my_type");
-        type.addField(TENSOR_FIELD, DataType.TENSOR);
+        type.addField(TENSOR_FIELD, new TensorDataType(tensorType));
         return type;
     }
 
     @Test
     public void requireThatTensorFieldValueIsSerializedAndDeserialized() {
-        assertSerialization(new TensorFieldValue());
-        assertSerialization(createTensor("{}"));
-        assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
+        assertSerialization(new TensorFieldValue(tensorType));
+        assertSerialization(createTensor(tensorType, "{}"));
+        assertSerialization(createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
     }
 
     @Test
     public void requireThatSerializationMatchesCpp() throws IOException {
-        assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue());
-        assertSerializationMatchesCpp("empty_tensor", createTensor("{}"));
+        assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue(tensorType));
+        assertSerializationMatchesCpp("empty_tensor", createTensor(tensorType, "{}"));
         assertSerializationMatchesCpp("multi_cell_tensor",
-                createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
+                createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
     }
 
     private static void assertSerialization(TensorFieldValue tensor) {
@@ -60,8 +63,8 @@ public class TensorFieldValueSerializationTestCase {
         SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory);
     }
 
-    private static TensorFieldValue createTensor(String tensor) {
-        return new TensorFieldValue(Tensor.from(tensor));
+    private static TensorFieldValue createTensor(TensorType type, String tensorCellString) {
+        return new TensorFieldValue(Tensor.from(type, tensorCellString));
     }
 
 }
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
index 3cfbcac5b62..3dc6ebd1403 100644
--- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
@@ -12,6 +12,7 @@ import com.yahoo.document.predicate.FeatureRange;
 import com.yahoo.document.predicate.FeatureSet;
 import com.yahoo.document.predicate.Predicate;
 import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.tensor.TensorType;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Test;
 
@@ -75,8 +76,8 @@ public class VespaXmlFieldReaderTestCase {
 
     @Test
     public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception {
-        assertThrows(new Field("my_tensor", DataType.TENSOR), "",
-                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+        assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "",
+                     "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
     }
 
     private class MockedReaderFixture {
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
index 8730265c80d..8a5fabde9ea 100644
--- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
@@ -7,7 +7,9 @@ import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.Field;
 import com.yahoo.document.StructDataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.tensor.TensorType;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -215,8 +217,8 @@ public class VespaXmlUpdateReaderTestCase {
 
     @Test
     public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception {
-        assertThrows(new Field("my_tensor", DataType.TENSOR), "<assign field='my_tensor'></assign>",
-                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+        assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "<assign field='my_tensor'></assign>",
+                     "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
     }
 
     private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp
index 365182b14eb..2c15c152558 100644
--- a/document/src/test/resources/tensor/empty_tensor__cpp
+++ b/document/src/test/resources/tensor/empty_tensor__cpp
diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java
index 1fbade8c785..2c15c152558 100644
--- a/document/src/test/resources/tensor/empty_tensor__java
+++ b/document/src/test/resources/tensor/empty_tensor__java