Revert "Revert "Revert "Bratseth/tensor type info in documents"""

16 files changed, 143 insertions, 196 deletions

diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java
index 4b24bb1ae00..640bd94bd1c 100644
--- a/document/src/main/java/com/yahoo/document/ArrayDataType.java
+++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java
@@ -8,10 +8,9 @@ import java.util.ArrayList;
 import java.util.List;
 
 /**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public class ArrayDataType extends CollectionDataType {
-
     // The global class identifier shared with C++.
     public static int classId = registerClass(Ids.document + 54, ArrayDataType.class);
 
diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java
index 184340f295e..51af799efd9 100644
--- a/document/src/main/java/com/yahoo/document/DataType.java
+++ b/document/src/main/java/com/yahoo/document/DataType.java
@@ -14,7 +14,6 @@ import com.yahoo.document.datatypes.Raw;
 import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.UriFieldValue;
-import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.objects.Identifiable;
 import com.yahoo.vespa.objects.Ids;
 import com.yahoo.vespa.objects.ObjectVisitor;
@@ -47,30 +46,40 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory());
     public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory());
     public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory());
+    // ARRAY is type 6, but never used, array IDs are generated
+    // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class);
     public final static DocumentType DOCUMENT = new DocumentType("document");
+    // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class);
     public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory());
+    // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class);
+    // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory());
     public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory());
-    public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory());
-    public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately 
-    // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor
-
-    // Tags are converted to weightedset<string> when reading the search definition TODO: Remove it
+    // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated
+    // Tags are converted to weightedset&lt;string&gt; when reading the search definition
     public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true);
+    // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class);
+    // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now..
+    // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType
+    public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory());
+    public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory());
 
     public static int lastPredefinedDataTypeId() {
         return 21;
     }
 
-    /** Set to true when this type is registered in a type manager. From that time we should refuse changes. */
+    /**
+     * Set to true when this type is registered in a type manager. From that time we should refuse changes.
+     */
     private boolean registered = false;
 
     private String name;
 
-    /** The id of this type */
+    /**
+     * The id of this type
+     */
     private int dataTypeId;
 
     static final private CopyOnWriteHashMap<Pair, Constructor> constructorCache = new CopyOnWriteHashMap<>();
-
     /**
      * Creates a datatype
      *
@@ -87,21 +96,26 @@ public abstract class DataType extends Identifiable implements Serializable, Com
         return (DataType)super.clone();
     }
 
-    void setRegistered() {
+    public void setRegistered() {
         registered = true;
     }
+
     public boolean isRegistered() {
         return registered;
     }
 
-    /** Creates a new, empty FieldValue of this type */
+    /**
+     * Creates a new, empty FieldValue of this type.
+     *
+     * @return a new, empty FieldValue of this type.
+     */
     public abstract FieldValue createFieldValue();
 
     /**
-     * Creates a field value by reflection
-     * 
-     * @param arg the value of the newly created field value
-     * @return a fully constructed value
+     * This will try to create the object by reflection. This can be very expensive
+     * so some might discourage that.
+     * @param arg The constructor argument.
+     * @return Fully constructed value.
      */
     protected FieldValue createByReflection(Object arg) {
         Class<?> valClass = getValueClass();
@@ -127,8 +141,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     /**
      * Creates a new FieldValue of this type, with the given value.
      *
-     * @param arg the value of the new FieldValue
-     * @return a new FieldValue of this type, with the given value
+     * @param arg the value that the new FieldValue shall have.
+     * @return A new FieldValue of this type, with the given value.
      */
     public FieldValue createFieldValue(Object arg) {
         if (arg == null) {
@@ -197,17 +211,12 @@ public abstract class DataType extends Identifiable implements Serializable, Com
         return new WeightedSetDataType(type, createIfNonExistent, removeIfZero);
     }
 
-    /** Returns the given tensor type as a DataType */
-    public static TensorDataType getTensor(TensorType type) {
-        return new TensorDataType(type);
-    }
-    
     public String getName() {
         return name;
     }
 
     /**
-     * Sets the name of this data type. WARNING! Do not use!
+     * Sets the name of this data type.&nbsp;WARNING! Do not use!
      *
      * @param name the name of this data type.
      */
@@ -220,7 +229,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     }
 
     /**
-     * Sets the ID of this data type. WARNING! Do not use!
+     * Sets the ID of this data type.&nbsp;WARNING! Do not use!
      *
      * @param id the ID of this data type.
      */
@@ -229,7 +238,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     }
 
     /**
-     * Registers this type in the given document manager.
+     * Registeres this type in the given document manager.
      *
      * @param manager the DocumentTypeManager to register in.
      */
@@ -246,12 +255,14 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     }
 
     public boolean equals(Object other) {
-        if (!(other instanceof DataType)) return false;
+        if (!(other instanceof DataType)) {
+            return false;
+        }
         DataType type = (DataType)other;
         return (name.equals(type.name) && dataTypeId == type.dataTypeId);
     }
 
-    public String toString() {
+    public java.lang.String toString() {
         return "datatype " + name + " (code: " + dataTypeId + ")";
     }
 
@@ -267,8 +278,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com
      */
     public FieldPath buildFieldPath(String fieldPathString) {
         if (fieldPathString.length() > 0) {
-            throw new IllegalArgumentException("Datatype " + toString() + 
-                                               " does not support further recursive structure: " + fieldPathString);
+            throw new IllegalArgumentException(
+                    "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString);
         }
         return new FieldPath();
     }
diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java
index e1d912b4e51..34c952e1cec 100644
--- a/document/src/main/java/com/yahoo/document/Document.java
+++ b/document/src/main/java/com/yahoo/document/Document.java
@@ -26,7 +26,7 @@ import java.util.Map;
  * be removed soon.
  *
  * @author bratseth
- * @author Einar M R Rosenvinge
+ * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public class Document extends StructuredFieldValue {
 
@@ -267,9 +267,7 @@ public class Document extends StructuredFieldValue {
     }
 
     /** Returns true if the argument is a document which has the same set of values */
-    @Override
     public boolean equals(Object o) {
-        if (o == this) return true;
         if (!(o instanceof Document)) return false;
         Document other = (Document) o;
         return (super.equals(o) && docId.equals(other.docId) &&
@@ -396,5 +394,4 @@ public class Document extends StructuredFieldValue {
         comp = body.compareTo(otherValue.body);
         return comp;
     }
-
 }
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
index 1b3370cfa66..0de7fe60500 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
@@ -12,7 +12,6 @@ import com.yahoo.document.serialization.DocumentDeserializer;
 import com.yahoo.document.serialization.DocumentDeserializerFactory;
 import com.yahoo.document.serialization.VespaDocumentDeserializer42;
 import com.yahoo.io.GrowableByteBuffer;
-import com.yahoo.tensor.TensorType;
 
 import java.lang.reflect.Modifier;
 import java.util.*;
@@ -29,23 +28,13 @@ import java.util.logging.Logger;
  * datatypes, for instance displaying the data type in human-readable form
  * or as XML.
  *
- * @author Thomas Gundersen
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
  */
 public class DocumentTypeManager {
 
     private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName());
     private ConfigSubscriber subscriber;
 
-    // *Configured data types* (not built-in/primitive) indexed by their id
-    //
-    // *Primitive* data types are always available and have a single id.
-    // 
-    // *Built-in dynamic* types: The tensor type.
-    // Any tensor type has the same id and is always available just like primitive types.
-    // However, unlike primitive types, each tensor type is a separate DataType instance
-    // (which carries additional type information (detailedType) supplied at runtime).
-    // The reason for this is that we want the data type to stay the same when we change the detailed tensor type
-    // to maintain compatibility on (some) tensor type changes.
     private Map<Integer, DataType> dataTypes = new LinkedHashMap<>();
     private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>();
     private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry();
@@ -96,7 +85,6 @@ public class DocumentTypeManager {
     }
 
     public boolean hasDataType(String name) {
-        if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present
         for (DataType type : dataTypes.values()) {
             if (type.getName().equalsIgnoreCase(name)) {
                 return true;
@@ -106,14 +94,10 @@ public class DocumentTypeManager {
     }
 
     public boolean hasDataType(int code) {
-        if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present
         return dataTypes.containsKey(code);
     }
 
     public DataType getDataType(String name) {
-        if (name.startsWith("tensor(")) // built-in dynamic
-            return new TensorDataType(TensorType.fromSpec(name));
-        
         List<DataType> foundTypes = new ArrayList<>();
         for (DataType type : dataTypes.values()) {
             if (type.getName().equalsIgnoreCase(name)) {
@@ -141,19 +125,7 @@ public class DocumentTypeManager {
         return foundTypes.get(0);
     }
 
-    public DataType getDataType(int code) { return getDataType(code, ""); }
-    
-    /**
-     * Return a data type instance
-     * 
-     * @param code the code of the data type to return, which must be either built in or present in this manager
-     * @param detailedType detailed type information, or the empty string if none
-     * @return the appropriate DataType instance
-     */
-    public DataType getDataType(int code, String detailedType) {
-        if (code == DataType.tensorDataTypeCode) // built-in dynamic
-            return new TensorDataType(TensorType.fromSpec(detailedType));
-
+    public DataType getDataType(int code) {
         DataType type = dataTypes.get(code);
         if (type == null) {
             StringBuilder types=new StringBuilder();
@@ -166,11 +138,11 @@ public class DocumentTypeManager {
         }
     }
 
-    DataType getDataTypeAndReturnTemporary(int code, String detailedType) {
+    DataType getDataTypeAndReturnTemporary(int code) {
         if (hasDataType(code)) {
-            return getDataType(code, detailedType);
+            return getDataType(code);
         }
-        return new TemporaryDataType(code, detailedType);
+        return new TemporaryDataType(code);
     }
 
     /**
@@ -184,11 +156,9 @@ public class DocumentTypeManager {
 
     /**
      * Register a single datatype. Re-registering an existing, but equal, datatype is ok.
-     * 
      * @param type The datatype to register
      */
     void registerSingleType(DataType type) {
-        if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly
         if (dataTypes.containsKey(type.getId())) {
             DataType existingType = dataTypes.get(type.getId());
             if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType))
@@ -218,7 +188,7 @@ public class DocumentTypeManager {
                 dataTypes.remove(existingType.getId());
             } else {
                 throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered "
-                                                + type + ", but already uses id " + type.getId());
+                        + type + ", but already uses id " + type.getId());
             }
         }
 
@@ -280,8 +250,7 @@ public class DocumentTypeManager {
     }
 
     /**
-     * Returns a read only view of the registered data types
-     * 
+     * A read only view of the registered data types
      * @return collection of types
      */
     public Collection<DataType> getDataTypes() {
@@ -349,7 +318,7 @@ public class DocumentTypeManager {
         for (Field field : structDataType.getFieldsThisTypeOnly()) {
             DataType fieldType = field.getDataType();
             if (fieldType instanceof TemporaryDataType) {
-                field.setDataType(getDataType(fieldType.getCode(), ((TemporaryDataType)fieldType).getDetailedType()));
+                field.setDataType(getDataType(fieldType.getCode()));
             } else {
                 if (!seenStructs.contains(fieldType)) {
                     replaceTemporaryTypes(fieldType, seenStructs);
@@ -360,7 +329,7 @@ public class DocumentTypeManager {
 
     private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List<DataType> seenStructs) {
         if (collectionDataType.getNestedType() instanceof TemporaryDataType) {
-            collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode(), ""));
+            collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode()));
         } else {
             replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs);
         }
@@ -368,13 +337,13 @@ public class DocumentTypeManager {
 
     private void replaceTemporaryTypesInMap(MapDataType mapDataType, List<DataType> seenStructs) {
         if (mapDataType.getValueType() instanceof TemporaryDataType) {
-            mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode(), ""));
+            mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode()));
         } else {
             replaceTemporaryTypes(mapDataType.getValueType(), seenStructs);
         }
 
         if (mapDataType.getKeyType() instanceof TemporaryDataType) {
-            mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode(), ""));
+            mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode()));
         } else {
             replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs);
         }
@@ -382,7 +351,7 @@ public class DocumentTypeManager {
 
     private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List<DataType> seenStructs) {
         if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) {
-            weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode(), ""));
+            weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode()));
         } else {
             replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs);
         }
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 7b5f1392032..9d7139e6226 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -13,7 +13,7 @@ import java.util.logging.Logger;
 /**
  * Configures the Vespa document manager from a config id.
  *
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{
 
@@ -82,21 +82,21 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                 try {
                     for (Object o : thisDataType.arraytype()) {
                         DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o;
-                        DataType nestedType = manager.getDataType(array.datatype(), "");
+                        DataType nestedType = manager.getDataType(array.datatype());
                         ArrayDataType type = new ArrayDataType(nestedType, id);
                         manager.register(type);
                     }
                     for (Object o : thisDataType.maptype()) {
                         DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o;
-                        DataType keyType = manager.getDataType(map.keytype(), "");
-                        DataType valType = manager.getDataType(map.valtype(), "");
+                        DataType keyType = manager.getDataType(map.keytype());
+                        DataType valType = manager.getDataType(map.valtype());
                         MapDataType type = new MapDataType(keyType, valType, id);
                         manager.register(type);
                     }
                     for (Object o : thisDataType.weightedsettype()) {
                         DocumentmanagerConfig.Datatype.Weightedsettype wset =
                                     (DocumentmanagerConfig.Datatype.Weightedsettype) o;
-                        DataType nestedType = manager.getDataType(wset.datatype(), "");
+                        DataType nestedType = manager.getDataType(wset.datatype());
                         WeightedSetDataType type = new WeightedSetDataType(
                                 nestedType, wset.createifnonexistant(), wset.removeifzero(), id);
                         manager.register(type);
@@ -114,8 +114,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                             DocumentmanagerConfig.Datatype.Structtype.Field field =
                                     (DocumentmanagerConfig.Datatype.Structtype.Field) j;
                             DataType fieldType = (field.datatype() == id)
-                                               ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype())
-                                               : manager.getDataType(field.datatype(), field.detailedtype());
+                                               ? manager.getDataTypeAndReturnTemporary(field.datatype())
+                                               : manager.getDataType(field.datatype());
 
                             if (field.id().size() == 1) {
                                 type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true));
@@ -127,8 +127,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                     }
                     for (Object o : thisDataType.documenttype()) {
                         DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o;
-                        StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), "");
-                        StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), "");
+                        StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct());
+                        StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct());
                         for (Field field : body.getFields()) {
                             field.setHeader(false);
                         }
@@ -210,7 +210,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
     private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) {
         for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) {
             AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id());
-            DataType payload = manager.getDataType(annType.datatype(), "");
+            DataType payload = manager.getDataType(annType.datatype());
             if (!payload.equals(DataType.NONE)) {
                 annotationType.setDataType(payload);
             }
@@ -234,7 +234,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             int id = thisDataType.id();
             for (Object o : thisDataType.structtype()) {
                 DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o;
-                StructDataType thisStruct = (StructDataType) manager.getDataType(id, "");
+                StructDataType thisStruct = (StructDataType) manager.getDataType(id);
 
                 for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) {
                     StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name());
diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java
index b80d92e0e4d..86543916b42 100644
--- a/document/src/main/java/com/yahoo/document/Field.java
+++ b/document/src/main/java/com/yahoo/document/Field.java
@@ -13,7 +13,7 @@ import java.io.Serializable;
  * A name and type. Fields are contained in document types to describe their fields,
  * but is also used to represent name/type pairs which are not part of document types.
  *
- * @author Thomas Gundersen
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
  * @author bratseth
  */
 public class Field extends FieldBase implements FieldSet, Comparable, Serializable {
diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
index 23bf4b43ccf..a0024bb0497 100644
--- a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
+++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
@@ -8,10 +8,9 @@ import com.yahoo.vespa.objects.ObjectVisitor;
 import java.util.Objects;
 
 /**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public class PrimitiveDataType extends DataType {
-
     public static abstract class Factory {
         public abstract FieldValue create();
     }
diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
index a0c6fcb889d..da65dde72da 100644
--- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java
+++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
@@ -4,15 +4,11 @@ package com.yahoo.document;
 import com.yahoo.document.datatypes.FieldValue;
 
 /**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 class TemporaryDataType extends DataType {
-
-    private final String detailedType;
-    
-    TemporaryDataType(int dataTypeId, String detailedType) {
+    TemporaryDataType(int dataTypeId) {
         super("temporary_" + dataTypeId, dataTypeId);
-        this.detailedType = detailedType;
     }
 
     @Override
@@ -29,7 +25,4 @@ class TemporaryDataType extends DataType {
     public boolean isValueCompatible(FieldValue value) {
         return false;
     }
-
-    String getDetailedType() { return detailedType; }
-
 }
diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java
deleted file mode 100644
index dbaf6ee7763..00000000000
--- a/document/src/main/java/com/yahoo/document/TensorDataType.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.yahoo.document;
-
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.datatypes.TensorFieldValue;
-import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.objects.Ids;
-
-/**
- * A DataType containing a tensor type
- * 
- * @author bratseth
- */
-public class TensorDataType extends DataType {
-
-    private final TensorType tensorType;
-    
-    // The global class identifier shared with C++.
-    public static int classId = registerClass(Ids.document + 59, TensorDataType.class);
-
-    public TensorDataType(TensorType tensorType) {
-        super(tensorType.toString(), DataType.tensorDataTypeCode);
-        this.tensorType = tensorType;
-    }
-
-    public TensorDataType clone() {
-        return (TensorDataType)super.clone();
-    }
-
-    @Override
-    public FieldValue createFieldValue() {
-        return new TensorFieldValue(tensorType);
-    }
-
-    @Override
-    public Class<? extends TensorFieldValue> getValueClass() {
-        return TensorFieldValue.class;
-    }
-
-    @Override
-    public boolean isValueCompatible(FieldValue value) {
-        if (value == null) return false;
-        if ( ! TensorFieldValue.class.isAssignableFrom(value.getClass())) return false;
-        TensorFieldValue tensorValue = (TensorFieldValue)value;
-        return tensorValue.getDataType().getTensorType().isAssignableTo(tensorType);
-    }
-
-    /** Returns the type of the tensor this field can hold */
-    public TensorType getTensorType() { return tensorType; }
-    
-}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java
index 09fb8b71db1..66cc472de69 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Array.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java
@@ -16,7 +16,7 @@ import java.util.*;
 /**
  * FieldValue which encapsulates a Array value
  *
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public final class Array<T extends FieldValue> extends CollectionFieldValue<T> implements List<T> {
 
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
index b5920cb4758..5a01dc33aa1 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
@@ -215,6 +215,7 @@ public class Struct extends StructuredFieldValue {
         if (!super.equals(o)) return false;
 
         Struct struct = (Struct) o;
+
         return values.equals(struct.values);
     }
 
@@ -387,5 +388,4 @@ public class Struct extends StructuredFieldValue {
         }
         return fieldType.cast(fieldValue);
     }
-
 }
diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
index 7957d9b812f..b4585a2188d 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
@@ -10,7 +10,7 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * @author Håkon Humberset
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
  */
 public abstract class StructuredFieldValue extends CompositeFieldValue {
 
diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
index 9d8e9a83b5e..bee29478219 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
@@ -3,12 +3,11 @@ package com.yahoo.document.datatypes;
 
 import com.yahoo.document.DataType;
 import com.yahoo.document.Field;
-import com.yahoo.document.TensorDataType;
+import com.yahoo.document.PrimitiveDataType;
 import com.yahoo.document.serialization.FieldReader;
 import com.yahoo.document.serialization.FieldWriter;
 import com.yahoo.document.serialization.XmlStream;
 import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -21,18 +20,12 @@ import java.util.Optional;
 public class TensorFieldValue extends FieldValue {
 
     private Optional<Tensor> tensor;
-    
-    private final TensorDataType dataType;
 
-    /** Create an empty tensor field value */
-    public TensorFieldValue(TensorType type) {
-        this.dataType = new TensorDataType(type);
-        this.tensor = Optional.empty();
+    public TensorFieldValue() {
+        tensor = Optional.empty();
     }
 
-    /** Create a tensor field value containing the given tensor */
     public TensorFieldValue(Tensor tensor) {
-        this.dataType = new TensorDataType(tensor.type());
         this.tensor = Optional.of(tensor);
     }
 
@@ -41,8 +34,8 @@ public class TensorFieldValue extends FieldValue {
     }
 
     @Override
-    public TensorDataType getDataType() {
-        return dataType;
+    public DataType getDataType() {
+        return DataType.TENSOR;
     }
 
     @Override
@@ -58,23 +51,16 @@ public class TensorFieldValue extends FieldValue {
     @Override
     public void assign(Object o) {
         if (o == null) {
-            assignTensor(Optional.empty());
+            tensor = Optional.empty();
         } else if (o instanceof Tensor) {
-            assignTensor(Optional.of((Tensor)o));
+            tensor = Optional.of((Tensor)o);
         } else if (o instanceof TensorFieldValue) {
-            assignTensor(((TensorFieldValue)o).getTensor());
+            tensor = ((TensorFieldValue)o).getTensor();
         } else {
             throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" +
-                                               o.getClass().getName() + "'.");
+                    o.getClass().getName() + "'.");
         }
     }
-    
-    public void assignTensor(Optional<Tensor> tensor) {
-        if (tensor.isPresent() && ! dataType.getTensorType().isAssignableTo(tensor.get().type()))
-            throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() +
-                                               " to field of type " + dataType.getTensorType());
-        this.tensor = tensor;
-    }
 
     @Override
     public void serialize(Field field, FieldWriter writer) {
@@ -88,14 +74,27 @@ public class TensorFieldValue extends FieldValue {
 
     @Override
     public boolean equals(Object o) {
-        if (this == o) return true;
-        if ( ! (o instanceof TensorFieldValue)) return false;
-
-        TensorFieldValue other = (TensorFieldValue)o;
-        if ( ! dataType.getTensorType().equals(other.dataType.getTensorType())) return false;
-        if ( ! tensor.equals(other.tensor)) return false;
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof TensorFieldValue)) {
+            return false;
+        }
+        TensorFieldValue rhs = (TensorFieldValue)o;
+        if (!Objects.equals(tensor, rhs.tensor)) {
+            return false;
+        }
         return true;
     }
 
+    public static PrimitiveDataType.Factory getFactory() {
+        return new PrimitiveDataType.Factory() {
+
+            @Override
+            public FieldValue create() {
+                return new TensorFieldValue();
+            }
+        };
+    }
 }
 
diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index e0740338aae..b1c635d7641 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -583,41 +583,71 @@ public class JsonReader {
     }
 
     private void fillTensor(TensorFieldValue tensorFieldValue) {
-        Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
         expectObjectStart(buffer.currentToken());
         int initNesting = buffer.nesting();
+        Tensor.Builder tensorBuilder = null;
         // read tensor cell fields and ignore everything else
         for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
             if (TENSOR_CELLS.equals(buffer.currentName()))
-                readTensorCells(tensorBuilder);
+                tensorBuilder = readTensorCells(tensorBuilder);
         }
         expectObjectEnd(buffer.currentToken());
+        if (tensorBuilder == null) // no cells + no type: empty tensor type
+            tensorBuilder = Tensor.Builder.of(TensorType.empty);
         tensorFieldValue.assign(tensorBuilder.build());
     }
 
-    private void readTensorCells(Tensor.Builder tensorBuilder) {
+    private Tensor.Builder readTensorCells(Tensor.Builder tensorBuilder) {
         expectArrayStart(buffer.currentToken());
         int initNesting = buffer.nesting();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next())
-            readTensorCell(tensorBuilder);
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            tensorBuilder = readTensorCell(tensorBuilder);
+        }
         expectCompositeEnd(buffer.currentToken());
+        return tensorBuilder;
     }
 
-    private void readTensorCell(Tensor.Builder tensorBuilder) {
+    private Tensor.Builder readTensorCell(Tensor.Builder tensorBuilder) {
         expectObjectStart(buffer.currentToken());
         int initNesting = buffer.nesting();
         double cellValue = 0.0;
-        Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell();
+        Tensor.Builder.CellBuilder cellBuilder = null;
         for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
             String currentName = buffer.currentName();
             if (TENSOR_ADDRESS.equals(currentName)) {
-                readTensorAddress(cellBuilder);
+                if (tensorBuilder != null) {
+                    cellBuilder = tensorBuilder.cell();
+                    readTensorAddress(cellBuilder);
+                }
+                else { // gnarly temporary path to create a type on the fly TODO; Remove when we always have a type
+                    expectObjectStart(buffer.currentToken());
+                    int initNesting2 = buffer.nesting();
+                    List<Pair<String,String>> entries = new ArrayList<>();
+                    for (buffer.next(); buffer.nesting() >= initNesting2; buffer.next()) {
+                        String dimension = buffer.currentName();
+                        String label = buffer.currentText();
+                        entries.add(new Pair<>(dimension, label));
+                    }
+                    TensorType.Builder typeBuilder = new TensorType.Builder();
+                    for (Pair<String,String> entry : entries)
+                        typeBuilder.mapped(entry.getFirst());
+                    tensorBuilder = Tensor.Builder.of(typeBuilder.build());
+                    cellBuilder = tensorBuilder.cell();
+                    for (Pair<String,String> entry : entries)
+                        cellBuilder.label(entry.getFirst(), entry.getSecond());
+                    expectObjectEnd(buffer.currentToken());
+                }
             } else if (TENSOR_VALUE.equals(currentName)) {
                 cellValue = Double.valueOf(buffer.currentText());
             }
         }
         expectObjectEnd(buffer.currentToken());
+        if (tensorBuilder == null) { // no content TODO; This will go away with the above
+            tensorBuilder = Tensor.Builder.of(TensorType.empty);
+            cellBuilder = tensorBuilder.cell();
+        }
         cellBuilder.value(cellValue);
+        return tensorBuilder;
     }
 
     private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) {
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
index 5f1b227790b..52a62caf296 100644
--- a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
@@ -9,7 +9,7 @@ import com.yahoo.document.DocumentTypeManager;
 /**
  * This interface is used to implement custom deserialization of document updates.
  *
- * @author Ravi Sharma
+ * @author <a href="mailto:ravishar@yahoo-inc.com">Ravi Sharma</a>
  * @author baldersheim
  */
 public interface DocumentReader {
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
index 6e9495b1437..753008de7e0 100644
--- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
@@ -115,8 +115,8 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
         // Verify that we have correct version
         version = getShort(null);
         if (version < 6 || version > Document.SERIALIZED_VERSION) {
-            throw new DeserializationException("Unknown version " + version + ", expected " + 
-                                               Document.SERIALIZED_VERSION + ".");
+            throw new DeserializationException(
+                    "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + ".");
         }
 
         int dataLength = 0;
@@ -278,7 +278,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
         int encodedTensorLength = buf.getInt1_4Bytes();
         if (encodedTensorLength > 0) {
             byte[] encodedTensor = getBytes(null, encodedTensorLength);
-            value.assign(TypedBinaryFormat.decode(value.getDataType().getTensorType(), encodedTensor));
+            value.assign(TypedBinaryFormat.decode(null, encodedTensor)); // TODO: Pass type
         } else {
             value.clear();
         }
@@ -328,7 +328,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
             fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null)));
         }
 
-        // save a reference to the big buffer we're reading from:
+        //save a reference to the big buffer we're reading from:
         GrowableByteBuffer bigBuf = buf;
 
         if (version < 7) {