Revert "Revert "Revert "Revert "Bratseth/tensor type info in documents""""

15 files changed, 200 insertions, 173 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
index 9c507d7be05..ff1a582d7fa 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
@@ -9,7 +9,9 @@ import java.util.Collection;
  * @author baldersheim
  */
 public interface DataTypeCollection {
-    public DataType getDataType(String name);
-    public DataType getDataType(int id);
-    public Collection<DataType> getTypes();
+
+    DataType getDataType(String name);
+    DataType getDataType(int id);
+    Collection<DataType> getTypes();
+
 }
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
index 73f8d7f91c5..2c527201ce4 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
@@ -11,6 +11,7 @@ import java.util.Map;
  * @author baldersheim
  */
 public class DataTypeRepo implements DataTypeCollection {
+
     Map<Integer, DataType> typeById = new LinkedHashMap<>();
     Map<String, DataType> typeByName = new LinkedHashMap<>();
 
@@ -25,10 +26,9 @@ public class DataTypeRepo implements DataTypeCollection {
     public Collection<DataType> getTypes() { return typeById.values(); }
 
     public DataTypeRepo add(DataType type) {
-        if (typeByName.containsKey(type.getName()) ||
-            typeById.containsKey(type.getId()))
-        {
-            throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered.");
+        if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) {
+            throw new IllegalStateException("Data type '" + type.getName() + "', id '" + 
+                                            type.getId() + "' is already registered.");
         }
         typeByName.put(type.getName(), type);
         typeById.put(type.getId(), type);
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
index b71a400666a..1abbf8b3ae1 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
@@ -9,6 +9,7 @@ import java.util.Map;
  * @author baldersheim
  */
 public class DocumentTypeRepo implements DocumentTypeCollection {
+
     final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>();
     final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>();
 
@@ -36,4 +37,5 @@ public class DocumentTypeRepo implements DocumentTypeCollection {
         typeById.put(type.getFullName().getId(), type);
         return this;
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index ccf9bdec302..7c594a4e836 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -13,15 +13,10 @@ import com.yahoo.searchdefinition.processing.BuiltInFieldSets;
 import java.util.*;
 
 /**
- * TODO: What is this and why?
- *
  * @author baldersheim
  */
 public final class NewDocumentType extends StructuredDataType implements DataTypeCollection {
 
-    /**
-     * TODO: What is this and why?
-     */
     public static final class Name {
 
         // TODO: privatize
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
index 793a5fcff6c..09aa55f776b 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
@@ -32,7 +32,6 @@ public class VespaDocumentType {
         vespa.add(PositionDataType.INSTANCE);
         vespa.add(DataType.URI);
         vespa.add(DataType.PREDICATE);
-        vespa.add(DataType.TENSOR);
         return vespa;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
index 7d8a87cee8c..2a723596255 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
@@ -11,9 +11,10 @@ import java.util.*;
 import java.util.logging.Level;
 
 /**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author Einar M R Rosenvinge
  */
 public class SDDocumentTypeOrderer {
+
     private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>();
     private Set<Integer> seenTypes = new LinkedHashSet<>();
     List<SDDocumentType> processingOrder = new LinkedList<>();
@@ -119,6 +120,8 @@ public class SDDocumentTypeOrderer {
             //do nothing
         } else if (type instanceof PrimitiveDataType) {
             //do nothing
+        } else if (type instanceof TensorDataType) {
+            //do nothing
         } else {
             deployLogger.log(Level.WARNING, "Unknown type : " + type);
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index 950ec791368..bc52aa27f4c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -25,10 +25,6 @@ public final class Attribute implements Cloneable, Serializable {
     private Type type;
     private CollectionType collectionType;
 
-    /** True if only the enum information should be read from this attribute
-     *  (i.e. the actual values are not relevant, only which documents have the
-     *   same values) Used for collapsing and unique.
-     */
     private boolean removeIfZero = false;
     private boolean createIfNonExistent = false;
     private boolean enableBitVectors = false;
@@ -41,6 +37,8 @@ public final class Attribute implements Cloneable, Serializable {
     private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND;
     private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND;
     private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD;
+    
+    /** This is set if the type of this is TENSOR */
     private Optional<TensorType> tensorType = Optional.empty();
 
     private boolean isPosition = false;
@@ -104,16 +102,21 @@ public final class Attribute implements Cloneable, Serializable {
     }
 
     /** Creates an attribute with default settings */
-    public Attribute(String name,DataType fieldType) {
-        this(name,convertDataType(fieldType), convertCollectionType(fieldType));
+    public Attribute(String name, DataType fieldType) {
+        this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType));
         setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false);
         setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false);
     }
 
-    public Attribute(String name,Type type, CollectionType collectionType) {
+    public Attribute(String name, Type type, CollectionType collectionType) {
+        this(name, type, collectionType, Optional.empty());
+    }
+
+    public Attribute(String name, Type type, CollectionType collectionType, Optional<TensorType> tensorType) {
         this.name=name;
         setType(type);
         setCollectionType(collectionType);
+        this.tensorType = tensorType;
     }
 
     /**
@@ -211,44 +214,42 @@ public final class Attribute implements Cloneable, Serializable {
     }
 
     /** Converts to the right attribute type from a field datatype */
-    public static CollectionType convertCollectionType(DataType fieldType) {
+    private static CollectionType convertCollectionType(DataType fieldType) {
         if (fieldType instanceof ArrayDataType) {
             return CollectionType.ARRAY;
         } else if (fieldType instanceof WeightedSetDataType) {
             return CollectionType.WEIGHTEDSET;
+        } else if (fieldType instanceof TensorDataType) {
+            return CollectionType.SINGLE;
         } else if (fieldType instanceof PrimitiveDataType) {
             return CollectionType.SINGLE;
         } else {
             throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType");
         }
     }
+    
+    private static Optional<TensorType> convertTensorType(DataType fieldType) {
+        if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty();
+        return Optional.of(((TensorDataType)fieldType).getTensorType());
+    }
 
     /** Converts to the right field type from an attribute type */
-    public static DataType convertAttrType(Type attrType) {
-        if (attrType== Type.STRING) {
-            return DataType.STRING;
-        } else if (attrType== Type.INTEGER) {
-            return DataType.INT;
-        } else if (attrType== Type.LONG) {
-            return DataType.LONG;
-        } else if (attrType== Type.FLOAT) {
-            return DataType.FLOAT;
-        } else if (attrType== Type.DOUBLE) {
-            return DataType.DOUBLE;
-        } else if (attrType == Type.BYTE) {
-            return DataType.BYTE;
-        } else if (attrType == Type.PREDICATE) {
-            return DataType.PREDICATE;
-        } else if (attrType == Type.TENSOR) {
-            return DataType.TENSOR;
-        } else {
-            throw new IllegalArgumentException("Don't know which attribute type to " +
-                    "convert " + attrType + " to");
+    private DataType toDataType(Type attributeType) {
+        switch (attributeType) {
+            case STRING : return DataType.STRING;
+            case INTEGER: return DataType.INT;
+            case LONG: return DataType.LONG;
+            case FLOAT: return DataType.FLOAT;
+            case DOUBLE: return DataType.DOUBLE;
+            case BYTE: return DataType.BYTE;
+            case PREDICATE: return DataType.PREDICATE;
+            case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new));
+            default: throw new IllegalArgumentException("Unknown attribute type " + attributeType);
         }
     }
 
     public DataType getDataType() {
-        DataType dataType = Attribute.convertAttrType(type);
+        DataType dataType = toDataType(type);
         if (collectionType.equals(Attribute.CollectionType.ARRAY)) {
             return DataType.getArray(dataType);
         } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
index 3b28f380b2d..78f0d9a2997 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
@@ -11,6 +11,7 @@ import com.yahoo.searchdefinition.document.SDField;
  */
 public interface FieldOperation extends Comparable<FieldOperation> {
 
+    /** Apply this operation on the given field */
     void apply(SDField field);
     
     @Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
index af04deb5347..d8b95391ecb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
@@ -16,7 +16,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 /**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author Simon Thoresen
  */
 public class IndexingValidation extends Processor {
 
@@ -26,11 +26,11 @@ public class IndexingValidation extends Processor {
 
     @Override
     public void process() {
-        VerificationContext ctx = new VerificationContext(new MyAdapter(search));
+        VerificationContext context = new VerificationContext(new MyAdapter(search));
         for (SDField field : search.allFieldsList()) {
             ScriptExpression script = field.getIndexingScript();
             try {
-                script.verify(ctx);
+                script.verify(context);
                 MyConverter converter = new MyConverter();
                 for (StatementExpression exp : script) {
                     converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch
@@ -123,8 +123,7 @@ public class IndexingValidation extends Processor {
                 throw new UnsupportedOperationException();
             }
             if (!fieldType.isAssignableFrom(valueType) &&
-                !fieldType.isAssignableFrom(createCompatType(valueType)))
-            {
+                !fieldType.isAssignableFrom(createCompatType(valueType))) {
                 throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
                                                      " '" + fieldName + "' which is " + fieldType.getName() + ".");
             }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index ae16f6cfed8..36cf302477e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -2,7 +2,9 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.CollectionDataType;
 import com.yahoo.document.DataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.Attribute;
@@ -23,12 +25,13 @@ public class TensorFieldProcessor extends Processor {
     @Override
     public void process() {
         for (SDField field : search.allFieldsList()) {
-            if (field.getDataType() == DataType.TENSOR) {
+            if ( field.getDataType() instanceof TensorDataType ) {
                 warnUseOfTensorFieldAsAttribute(field);
                 validateIndexingScripsForTensorField(field);
                 validateAttributeSettingForTensorField(field);
-            } else {
-                validateDataTypeForField(field);
+            }
+            else if (field.getDataType() instanceof CollectionDataType){
+                validateDataTypeForCollectionField(field);
             }
         }
     }
@@ -55,9 +58,9 @@ public class TensorFieldProcessor extends Processor {
         }
     }
 
-    private void validateDataTypeForField(SDField field) {
-        if (field.getDataType().getPrimitiveType() == DataType.TENSOR) {
+    private void validateDataTypeForCollectionField(SDField field) {
+        if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType)
             fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
-        }
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
index 1349abc3795..9ee5f48a906 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.document.DataType;
@@ -49,22 +50,25 @@ public class ValidateFieldTypes extends Processor {
         DataType seenType = seenFields.get(fieldName);
         if (seenType == null) {
             seenFields.put(fieldName, fieldType);
-        } else if ( ! equalTypes(seenType, fieldType)) {
-            throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + 
+        } else if ( ! compatibleTypes(seenType, fieldType)) {
+            throw newProcessException(searchName, fieldName, "Incompatible types. Expected " + 
                                                              seenType.getName() + " for " + fieldDesc +
                                                              " '" + fieldName + "', got " + fieldType.getName() + ".");
         }
     }
 
-    private boolean equalTypes(DataType d1, DataType d2) {
+    private boolean compatibleTypes(DataType seenType, DataType fieldType) {
         // legacy tag field type compatibility; probably not needed any more (Oct 2016)
-        if ("tag".equals(d1.getName())) {
-            return "tag".equals(d2.getName()) || "WeightedSet<string>".equals(d2.getName());
+        if ("tag".equals(seenType.getName())) {
+            return "tag".equals(fieldType.getName()) || "WeightedSet<string>".equals(fieldType.getName());
         }
-        if ("tag".equals(d2.getName())) {
-            return "tag".equals(d1.getName()) || "WeightedSet<string>".equals(d1.getName());
+        if ("tag".equals(fieldType.getName())) {
+            return "tag".equals(seenType.getName()) || "WeightedSet<string>".equals(seenType.getName());
         }
-        return d1.equals(d2);
+        if (seenType instanceof TensorDataType && fieldType instanceof TensorDataType) {
+            return fieldType.isAssignableFrom(seenType); // TODO: Just do this for all types
+        }
+        return seenType.equals(fieldType);
     }
     
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 3cdcf55b34e..705ec4dc4a1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -17,39 +17,39 @@ import java.util.Set;
 
 /**
  * @author    baldersheim
- * @since     2010-02-19
  */
 public class DocumentManager {
 
-    public DocumentmanagerConfig.Builder produce(DocumentModel model, DocumentmanagerConfig.Builder docman) {
-        docman.enablecompression(false);
+    public DocumentmanagerConfig.Builder produce(DocumentModel model, 
+                                                 DocumentmanagerConfig.Builder documentConfigBuilder) {
+        documentConfigBuilder.enablecompression(false);
         Set<DataType> handled = new HashSet<>();
         for(NewDocumentType documentType : model.getDocumentManager().getTypes()) {
-            handle(documentType, docman, handled);
-            handleAnnotations(documentType.getAnnotations(), docman);
+            buildConfig(documentType, documentConfigBuilder, handled);
+            buildConfig(documentType.getAnnotations(), documentConfigBuilder);
             if ( documentType != VespaDocumentType.INSTANCE) {
-                DocumentmanagerConfig.Datatype.Builder dt = new DocumentmanagerConfig.Datatype.Builder();
-                docman.datatype(dt);
-                handleDataType(documentType, dt);
+                DocumentmanagerConfig.Datatype.Builder dataTypeBuilder = new DocumentmanagerConfig.Datatype.Builder();
+                documentConfigBuilder.datatype(dataTypeBuilder);
+                buildConfig(documentType, dataTypeBuilder);
             }
         }
-        return docman;
+        return documentConfigBuilder;
     }
 
-    private void handle(DataTypeCollection type, DocumentmanagerConfig.Builder docman, Set<DataType> handled) {
+    private void buildConfig(DataTypeCollection type, DocumentmanagerConfig.Builder documentConfigBuilder, Set<DataType> built) {
         for (DataType dataType : type.getTypes()) {
-            if (handled.contains(dataType)) continue;
-            handled.add(dataType);
+            if (built.contains(dataType)) continue;
+            built.add(dataType);
             if (dataType instanceof TemporaryStructuredDataType) continue;
-            if ((dataType.getId() < 0) || (DataType.lastPredefinedDataTypeId() < dataType.getId())) {
-                Datatype.Builder dtc = new Datatype.Builder();
-                docman.datatype(dtc);
-                handleDataType(dataType, dtc);
+            if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) {
+                Datatype.Builder dataTypeBuilder = new Datatype.Builder();
+                documentConfigBuilder.datatype(dataTypeBuilder);
+                buildConfig(dataType, dataTypeBuilder);
             }
         }
     }
 
-    private void handleAnnotation(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) {
+    private void buildConfig(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) {
         atb.
             id(type.getId()).
             name(type.getName());
@@ -62,34 +62,34 @@ public class DocumentManager {
             }
         }
     }
-    private void handleAnnotations(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
+    private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
         for (AnnotationType type : types) {
             DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
-            handleAnnotation(type, atb);
+            buildConfig(type, atb);
             builder.annotationtype(atb);
         }
     }
 
-    private void handleDataType(DataType type, Datatype.Builder dtc) {
-        dtc.id(type.getId());
+    private void buildConfig(DataType type, Datatype.Builder builder) {
+        builder.id(type.getId());
         if (type instanceof ArrayDataType) {
             CollectionDataType dt = (CollectionDataType) type;
-            dtc.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId()));
+            builder.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId()));
         } else if (type instanceof WeightedSetDataType) {
             WeightedSetDataType dt = (WeightedSetDataType) type;
-            dtc.weightedsettype(new Datatype.Weightedsettype.Builder().
+            builder.weightedsettype(new Datatype.Weightedsettype.Builder().
                     datatype(dt.getNestedType().getId()).
                     createifnonexistant(dt.createIfNonExistent()).
                     removeifzero(dt.removeIfZero()));
         } else if (type instanceof MapDataType) {
             MapDataType mtype = (MapDataType) type;
-            dtc.maptype(new Datatype.Maptype.Builder().
+            builder.maptype(new Datatype.Maptype.Builder().
                     keytype(mtype.getKeyType().getId()).
                     valtype(mtype.getValueType().getId()));
         } else if (type instanceof DocumentType) {
             DocumentType dt = (DocumentType) type;
             Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
-            dtc.documenttype(doc);
+            builder.documenttype(doc);
             doc.
                 name(dt.getName()).
                 headerstruct(dt.getHeaderType().getId()).
@@ -100,7 +100,7 @@ public class DocumentManager {
         } else if (type instanceof NewDocumentType) {
             NewDocumentType dt = (NewDocumentType) type;
             Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
-            dtc.documenttype(doc);
+            builder.documenttype(doc);
             doc.
                 name(dt.getName()).
                 headerstruct(dt.getHeader().getId()).
@@ -108,49 +108,56 @@ public class DocumentManager {
             for (NewDocumentType inherited : dt.getInherited()) {
                 doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
             }
-            handleFieldSets(dt.getFieldSets(), doc);
+            buildConfig(dt.getFieldSets(), doc);
         } else if (type instanceof TemporaryStructuredDataType) {
             //Ignored
         } else if (type instanceof StructDataType) {
-            StructDataType dt = (StructDataType) type;
-            Datatype.Structtype.Builder st = new Datatype.Structtype.Builder();
-            dtc.structtype(st);
-            st.name(dt.getName());
-            if (dt.getCompressionConfig().type.getCode() != 0) {
-                st.
-                    compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(dt.getCompressionConfig().type.toString())).
-                    compresslevel(dt.getCompressionConfig().compressionLevel).
-                    compressthreshold((int)dt.getCompressionConfig().threshold).
-                    compressminsize((int)dt.getCompressionConfig().minsize);
+            StructDataType structType = (StructDataType) type;
+            Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder();
+            builder.structtype(structBuilder);
+            structBuilder.name(structType.getName());
+            if (structType.getCompressionConfig().type.getCode() != 0) {
+                structBuilder.
+                    compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(structType.getCompressionConfig().type.toString())).
+                    compresslevel(structType.getCompressionConfig().compressionLevel).
+                    compressthreshold((int)structType.getCompressionConfig().threshold).
+                    compressminsize((int)structType.getCompressionConfig().minsize);
             }
-            for (com.yahoo.document.Field field : dt.getFieldsThisTypeOnly()) {
-                Datatype.Structtype.Field.Builder fb = new Datatype.Structtype.Field.Builder();
-                st.field(fb);
-                fb.name(field.getName());
+            for (com.yahoo.document.Field field : structType.getFieldsThisTypeOnly()) {
+                Datatype.Structtype.Field.Builder fieldBuilder = new Datatype.Structtype.Field.Builder();
+                structBuilder.field(fieldBuilder);
+                fieldBuilder.name(field.getName());
                 if (field.hasForcedId()) {
-                    fb.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId()));
+                    fieldBuilder.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId()));
                 }
-                fb.datatype(field.getDataType().getId());
+                fieldBuilder.datatype(field.getDataType().getId());
+
+                if (field.getDataType() instanceof TensorDataType)
+                    fieldBuilder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString());
             }
-            for (StructDataType inherited : dt.getInheritedTypes()) {
-                st.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName()));
+            for (StructDataType inherited : structType.getInheritedTypes()) {
+                structBuilder.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName()));
             }
         } else if (type instanceof AnnotationReferenceDataType) {
             AnnotationReferenceDataType annotationRef = (AnnotationReferenceDataType) type;
-            dtc.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName()));
+            builder.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName()));
+        } else if (type instanceof TensorDataType) {
+            // Nothing to do; the type of the tensor is instead stored in each field as detailed type information
+            // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
+            // without changing the field type and thus requiring data refeed
         } else {
-            throw new IllegalArgumentException("Can not handle datatype '" + type.getName());
+            throw new IllegalArgumentException("Can not create config for data type '" + type.getName());
         }
     }
 
-    private void handleFieldSets(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) {
-
+    private void buildConfig(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) {
         for (FieldSet builtinFs : fieldSets) {
-            handleFieldSet(builtinFs, doc);
+            buildConfig(builtinFs, doc);
         }
     }
 
-    private void handleFieldSet(FieldSet fs, Datatype.Documenttype.Builder doc) {
+    private void buildConfig(FieldSet fs, Datatype.Documenttype.Builder doc) {
         doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index 2ba34def464..ebe18b36410 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -2,13 +2,11 @@
 package com.yahoo.vespa.configmodel.producers;
 
 import com.yahoo.document.*;
-import com.yahoo.document.DocumenttypesConfig.Builder;
 import com.yahoo.document.annotation.AnnotationReferenceDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.documentmodel.DataTypeCollection;
 import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.documentmodel.VespaDocumentType;
-import com.yahoo.searchdefinition.FieldSets;
 import com.yahoo.searchdefinition.document.FieldSet;
 import com.yahoo.vespa.documentmodel.DocumentModel;
 import java.util.*;
@@ -31,12 +29,12 @@ public class DocumentTypes {
             for (NewDocumentType inherited : documentType.getInherited()) {
                 produceInheritOrder(inherited, builder, produced);
             }
-            handle(documentType, builder);
+            buildConfig(documentType, builder);
             produced.put(documentType.getFullName(), documentType);
         }
     }
 
-    private void handle(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
+    private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
         if (documentType == VespaDocumentType.INSTANCE) {
             return;
         }
@@ -46,40 +44,40 @@ public class DocumentTypes {
             name(documentType.getName()).
             headerstruct(documentType.getHeader().getId()).
             bodystruct(documentType.getBody().getId());
-        Set<Integer> handled = new HashSet<>();
+        Set<Integer> built = new HashSet<>();
         for (NewDocumentType inherited : documentType.getInherited()) {
             db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId()));
-            markAsHandled(handled, inherited.getAllTypes());
+            markAsBuilt(built, inherited.getAllTypes());
         }
         for (DataType dt : documentType.getTypes()) {
-            handle(dt, db, handled);
+            buildConfig(dt, db, built);
         }
         for(AnnotationType annotation : documentType.getAnnotations()) {
             DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder();
             db.annotationtype(atb);
-            handle(annotation, atb);
+            buildConfig(annotation, atb);
         }
-        handleFieldSets(documentType.getFieldSets(), db);
+        buildConfig(documentType.getFieldSets(), db);
         builder.documenttype(db);
     }
 
-    private void handleFieldSets(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) {
+    private void buildConfig(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) {
         for (FieldSet fs : fieldSets) {
-            handleFieldSet(fs, db);
+            buildConfig(fs, db);
         }        
     }
 
-    private void handleFieldSet(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
+    private void buildConfig(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
         db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
 
-    private void markAsHandled(Set<Integer> handled, DataTypeCollection typeCollection) {
+    private void markAsBuilt(Set<Integer> built, DataTypeCollection typeCollection) {
         for (DataType type : typeCollection.getTypes()) {
-            handled.add(type.getId());
+            built.add(type.getId());
         }
     }
 
-    private void handle(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
+    private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
         builder.
             id(annotation.getId()).
             name(annotation.getName());
@@ -92,67 +90,77 @@ public class DocumentTypes {
         }
     }
 
-    private void handle(DataType type, DocumenttypesConfig.Documenttype.Builder db, Set<Integer> handled) {
-        if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! handled.contains(type.getId())) {
-            handled.add(type.getId());
-            DocumenttypesConfig.Documenttype.Datatype.Builder dtb = new DocumenttypesConfig.Documenttype.Datatype.Builder();
-            dtb.id(type.getId());
+    private void buildConfig(DataType type, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) {
+        if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! built.contains(type.getId())) {
+            built.add(type.getId());
+            DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder();
+            dataTypeBuilder.id(type.getId());
             if (type instanceof StructDataType) {
-                dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("STRUCT"));
+                dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT);
                 StructDataType dt = (StructDataType) type;
-                DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder sb = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
-                dtb.sstruct(sb);
-                sb.name(dt.getName());
+                DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
+                dataTypeBuilder.sstruct(structBuilder);
+                structBuilder.name(dt.getName());
                 if (dt.getCompressionConfig().type.getCode() != 0) {
-                    sb.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder().
+                    structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder().
                             type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(dt.getCompressionConfig().type.toString())).
                             level(dt.getCompressionConfig().compressionLevel).
                             threshold((int)dt.getCompressionConfig().threshold).
                             minsize((int)dt.getCompressionConfig().minsize));
                 }
                 for (com.yahoo.document.Field field : dt.getFields()) {
-                    sb.field(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder().
-                            name(field.getName()).
+                    DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder =
+                            new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder();
+                    builder.name(field.getName()).
                             id(field.getId()).
                             id_v6(field.getIdV6()).
-                            datatype(field.getDataType().getId()));
-                    handle(field.getDataType(), db, handled);
+                            datatype(field.getDataType().getId());
+                    if (field.getDataType() instanceof TensorDataType)
+                        builder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString());
+                    structBuilder.field(builder);
+                    buildConfig(field.getDataType(), documentBuilder, built);
                 }
             } else if (type instanceof ArrayDataType) {
-                dtb.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ARRAY")).
+                dataTypeBuilder.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY).
                     array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder().
                             element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(((ArrayDataType)type).getNestedType().getId())));
-                handle(((ArrayDataType)type).getNestedType(), db, handled);
+                buildConfig(((ArrayDataType)type).getNestedType(), documentBuilder, built);
             } else if (type instanceof WeightedSetDataType) {
-                dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("WSET")).
+                dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET).
                 wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder().
                         key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder().
                                 id(((WeightedSetDataType)type).getNestedType().getId())).
                         createifnonexistent(((WeightedSetDataType)type).createIfNonExistent()).
                         removeifzero(((WeightedSetDataType)type).removeIfZero()));
-                handle(((WeightedSetDataType)type).getNestedType(), db, handled);
+                buildConfig(((WeightedSetDataType)type).getNestedType(), documentBuilder, built);
             } else if (type instanceof MapDataType) {
-                dtb.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("MAP")).
+                dataTypeBuilder.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP).
                     map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder().
                         key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder().
                             id(((MapDataType)type).getKeyType().getId())).
                         value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder().
                             id(((MapDataType)type).getValueType().getId())));
-                handle(((MapDataType)type).getKeyType(), db, handled);
-                handle(((MapDataType)type).getValueType(), db, handled);
+                buildConfig(((MapDataType)type).getKeyType(), documentBuilder, built);
+                buildConfig(((MapDataType)type).getValueType(), documentBuilder, built);
             } else if (type instanceof AnnotationReferenceDataType) {
-                dtb.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ANNOTATIONREF")).
+                dataTypeBuilder.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ANNOTATIONREF).
                     annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder().
                             annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder().
                                     id(((AnnotationReferenceDataType)type).getAnnotationType().getId())));
+            } else if (type instanceof TensorDataType) {
+                // The type of the tensor is not stored here but instead in each field as detailed type information
+                // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
+                // without changing the field type and thus requiring data refeed
+                return;
             } else {
                 return;
             }
-            db.datatype(dtb);
+            documentBuilder.datatype(dataTypeBuilder);
         }
     }
+
 }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index c03fb0617b8..1d39d1e6928 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -43,7 +43,7 @@ public class SearchDataTypeValidator extends Validator {
         for (Field field : doc.fieldSet()) {
             DataType fieldType = field.getDataType();
             disallowIndexingOfMaps(cluster, def, field);
-            if (!validateDataType(fieldType)) {
+            if ( ! isSupportedInSearchClusters(fieldType)) {
                 throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " +
                                                    "clusters (field '" + field.getName() + "' in definition '" +
                                                    def.getName() + "' for cluster '" + cluster.getClusterName() + "').");
@@ -51,28 +51,30 @@ public class SearchDataTypeValidator extends Validator {
         }
     }
 
-    private boolean validateDataType(DataType dataType) {
-        if (dataType instanceof ArrayDataType ||
-            dataType instanceof WeightedSetDataType)
-        {
-            return validateDataType(((CollectionDataType)dataType).getNestedType());
+    private boolean isSupportedInSearchClusters(DataType dataType) {
+        if (dataType instanceof ArrayDataType || dataType instanceof WeightedSetDataType) {
+            return isSupportedInSearchClusters(((CollectionDataType)dataType).getNestedType());
         }
-        if (dataType instanceof StructDataType) {
+        else if (dataType instanceof StructDataType) {
             return true; // Struct will work for summary TODO maybe check individual fields
         }
-        if (dataType instanceof MapDataType) {
+        else if (dataType instanceof MapDataType) {
             return true; // Maps will work for summary, see disallowIndexingOfMaps()
         }
-        return dataType.equals(DataType.INT) ||
-               dataType.equals(DataType.FLOAT) ||
-               dataType.equals(DataType.STRING) ||
-               dataType.equals(DataType.RAW) ||
-               dataType.equals(DataType.LONG) ||
-               dataType.equals(DataType.DOUBLE) ||
-               dataType.equals(DataType.URI) ||
-               dataType.equals(DataType.BYTE) ||
-               dataType.equals(DataType.PREDICATE) ||
-               dataType.equals(DataType.TENSOR);
+        else if (dataType instanceof TensorDataType) {
+            return true;
+        }
+        else {
+            return dataType.equals(DataType.INT) ||
+                   dataType.equals(DataType.FLOAT) ||
+                   dataType.equals(DataType.STRING) ||
+                   dataType.equals(DataType.RAW) ||
+                   dataType.equals(DataType.LONG) ||
+                   dataType.equals(DataType.DOUBLE) ||
+                   dataType.equals(DataType.URI) ||
+                   dataType.equals(DataType.BYTE) ||
+                   dataType.equals(DataType.PREDICATE);
+        }
     }
 
     private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 32be8906ffc..3fc84f2860b 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -805,13 +805,14 @@ DataType dataType() :
     DataType mapType=null;
 	DataType arrayType=null;
     DataType wsetType=null;
-
+    TensorType tensorType;
 }
 {
     (   LOOKAHEAD(<ARRAY> <LESSTHAN>)       ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } )
       | LOOKAHEAD(<WEIGHTEDSET> <LESSTHAN>) ( <WEIGHTEDSET> <LESSTHAN> wsetType = dataType() <GREATERTHAN> { return  DataType.getWeightedSet(wsetType); } )
       | LOOKAHEAD(<MAP> <LESSTHAN>)         ( mapType = mapDataType() { return mapType; } )
       | LOOKAHEAD(<ANNOTATIONREFERENCE> <LESSTHAN>) ( mapType = annotationRefDataType() { return mapType; } )
+      | LOOKAHEAD(<TENSOR_TYPE>)                 ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } )
       | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] )
     )
     {