Revert "Bratseth/tensor type info in documents"

15 files changed, 173 insertions, 200 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
index ff1a582d7fa..9c507d7be05 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
@@ -9,9 +9,7 @@ import java.util.Collection;
  * @author baldersheim
  */
 public interface DataTypeCollection {
-
-    DataType getDataType(String name);
-    DataType getDataType(int id);
-    Collection<DataType> getTypes();
-
+    public DataType getDataType(String name);
+    public DataType getDataType(int id);
+    public Collection<DataType> getTypes();
 }
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
index 2c527201ce4..73f8d7f91c5 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
@@ -11,7 +11,6 @@ import java.util.Map;
  * @author baldersheim
  */
 public class DataTypeRepo implements DataTypeCollection {
-
     Map<Integer, DataType> typeById = new LinkedHashMap<>();
     Map<String, DataType> typeByName = new LinkedHashMap<>();
 
@@ -26,9 +25,10 @@ public class DataTypeRepo implements DataTypeCollection {
     public Collection<DataType> getTypes() { return typeById.values(); }
 
     public DataTypeRepo add(DataType type) {
-        if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) {
-            throw new IllegalStateException("Data type '" + type.getName() + "', id '" + 
-                                            type.getId() + "' is already registered.");
+        if (typeByName.containsKey(type.getName()) ||
+            typeById.containsKey(type.getId()))
+        {
+            throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered.");
         }
         typeByName.put(type.getName(), type);
         typeById.put(type.getId(), type);
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
index 1abbf8b3ae1..b71a400666a 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
@@ -9,7 +9,6 @@ import java.util.Map;
  * @author baldersheim
  */
 public class DocumentTypeRepo implements DocumentTypeCollection {
-
     final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>();
     final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>();
 
@@ -37,5 +36,4 @@ public class DocumentTypeRepo implements DocumentTypeCollection {
         typeById.put(type.getFullName().getId(), type);
         return this;
     }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index 7c594a4e836..ccf9bdec302 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -13,10 +13,15 @@ import com.yahoo.searchdefinition.processing.BuiltInFieldSets;
 import java.util.*;
 
 /**
+ * TODO: What is this and why?
+ *
  * @author baldersheim
  */
 public final class NewDocumentType extends StructuredDataType implements DataTypeCollection {
 
+    /**
+     * TODO: What is this and why?
+     */
     public static final class Name {
 
         // TODO: privatize
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
index 09aa55f776b..793a5fcff6c 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
@@ -32,6 +32,7 @@ public class VespaDocumentType {
         vespa.add(PositionDataType.INSTANCE);
         vespa.add(DataType.URI);
         vespa.add(DataType.PREDICATE);
+        vespa.add(DataType.TENSOR);
         return vespa;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
index 2a723596255..7d8a87cee8c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
@@ -11,10 +11,9 @@ import java.util.*;
 import java.util.logging.Level;
 
 /**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
  */
 public class SDDocumentTypeOrderer {
-
     private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>();
     private Set<Integer> seenTypes = new LinkedHashSet<>();
     List<SDDocumentType> processingOrder = new LinkedList<>();
@@ -120,8 +119,6 @@ public class SDDocumentTypeOrderer {
             //do nothing
         } else if (type instanceof PrimitiveDataType) {
             //do nothing
-        } else if (type instanceof TensorDataType) {
-            //do nothing
         } else {
             deployLogger.log(Level.WARNING, "Unknown type : " + type);
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index bc52aa27f4c..950ec791368 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -25,6 +25,10 @@ public final class Attribute implements Cloneable, Serializable {
     private Type type;
     private CollectionType collectionType;
 
+    /** True if only the enum information should be read from this attribute
+     *  (i.e. the actual values are not relevant, only which documents have the
+     *   same values) Used for collapsing and unique.
+     */
     private boolean removeIfZero = false;
     private boolean createIfNonExistent = false;
     private boolean enableBitVectors = false;
@@ -37,8 +41,6 @@ public final class Attribute implements Cloneable, Serializable {
     private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND;
     private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND;
     private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD;
-    
-    /** This is set if the type of this is TENSOR */
     private Optional<TensorType> tensorType = Optional.empty();
 
     private boolean isPosition = false;
@@ -102,21 +104,16 @@ public final class Attribute implements Cloneable, Serializable {
     }
 
     /** Creates an attribute with default settings */
-    public Attribute(String name, DataType fieldType) {
-        this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType));
+    public Attribute(String name,DataType fieldType) {
+        this(name,convertDataType(fieldType), convertCollectionType(fieldType));
         setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false);
         setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false);
     }
 
-    public Attribute(String name, Type type, CollectionType collectionType) {
-        this(name, type, collectionType, Optional.empty());
-    }
-
-    public Attribute(String name, Type type, CollectionType collectionType, Optional<TensorType> tensorType) {
+    public Attribute(String name,Type type, CollectionType collectionType) {
         this.name=name;
         setType(type);
         setCollectionType(collectionType);
-        this.tensorType = tensorType;
     }
 
     /**
@@ -214,42 +211,44 @@ public final class Attribute implements Cloneable, Serializable {
     }
 
     /** Converts to the right attribute type from a field datatype */
-    private static CollectionType convertCollectionType(DataType fieldType) {
+    public static CollectionType convertCollectionType(DataType fieldType) {
         if (fieldType instanceof ArrayDataType) {
             return CollectionType.ARRAY;
         } else if (fieldType instanceof WeightedSetDataType) {
             return CollectionType.WEIGHTEDSET;
-        } else if (fieldType instanceof TensorDataType) {
-            return CollectionType.SINGLE;
         } else if (fieldType instanceof PrimitiveDataType) {
             return CollectionType.SINGLE;
         } else {
             throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType");
         }
     }
-    
-    private static Optional<TensorType> convertTensorType(DataType fieldType) {
-        if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty();
-        return Optional.of(((TensorDataType)fieldType).getTensorType());
-    }
 
     /** Converts to the right field type from an attribute type */
-    private DataType toDataType(Type attributeType) {
-        switch (attributeType) {
-            case STRING : return DataType.STRING;
-            case INTEGER: return DataType.INT;
-            case LONG: return DataType.LONG;
-            case FLOAT: return DataType.FLOAT;
-            case DOUBLE: return DataType.DOUBLE;
-            case BYTE: return DataType.BYTE;
-            case PREDICATE: return DataType.PREDICATE;
-            case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new));
-            default: throw new IllegalArgumentException("Unknown attribute type " + attributeType);
+    public static DataType convertAttrType(Type attrType) {
+        if (attrType== Type.STRING) {
+            return DataType.STRING;
+        } else if (attrType== Type.INTEGER) {
+            return DataType.INT;
+        } else if (attrType== Type.LONG) {
+            return DataType.LONG;
+        } else if (attrType== Type.FLOAT) {
+            return DataType.FLOAT;
+        } else if (attrType== Type.DOUBLE) {
+            return DataType.DOUBLE;
+        } else if (attrType == Type.BYTE) {
+            return DataType.BYTE;
+        } else if (attrType == Type.PREDICATE) {
+            return DataType.PREDICATE;
+        } else if (attrType == Type.TENSOR) {
+            return DataType.TENSOR;
+        } else {
+            throw new IllegalArgumentException("Don't know which attribute type to " +
+                    "convert " + attrType + " to");
         }
     }
 
     public DataType getDataType() {
-        DataType dataType = toDataType(type);
+        DataType dataType = Attribute.convertAttrType(type);
         if (collectionType.equals(Attribute.CollectionType.ARRAY)) {
             return DataType.getArray(dataType);
         } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
index 78f0d9a2997..3b28f380b2d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
@@ -11,7 +11,6 @@ import com.yahoo.searchdefinition.document.SDField;
  */
 public interface FieldOperation extends Comparable<FieldOperation> {
 
-    /** Apply this operation on the given field */
     void apply(SDField field);
     
     @Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
index d8b95391ecb..af04deb5347 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
@@ -16,7 +16,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 /**
- * @author Simon Thoresen
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
  */
 public class IndexingValidation extends Processor {
 
@@ -26,11 +26,11 @@ public class IndexingValidation extends Processor {
 
     @Override
     public void process() {
-        VerificationContext context = new VerificationContext(new MyAdapter(search));
+        VerificationContext ctx = new VerificationContext(new MyAdapter(search));
         for (SDField field : search.allFieldsList()) {
             ScriptExpression script = field.getIndexingScript();
             try {
-                script.verify(context);
+                script.verify(ctx);
                 MyConverter converter = new MyConverter();
                 for (StatementExpression exp : script) {
                     converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch
@@ -123,7 +123,8 @@ public class IndexingValidation extends Processor {
                 throw new UnsupportedOperationException();
             }
             if (!fieldType.isAssignableFrom(valueType) &&
-                !fieldType.isAssignableFrom(createCompatType(valueType))) {
+                !fieldType.isAssignableFrom(createCompatType(valueType)))
+            {
                 throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
                                                      " '" + fieldName + "' which is " + fieldType.getName() + ".");
             }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 36cf302477e..ae16f6cfed8 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -2,9 +2,7 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.document.CollectionDataType;
 import com.yahoo.document.DataType;
-import com.yahoo.document.TensorDataType;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.Attribute;
@@ -25,13 +23,12 @@ public class TensorFieldProcessor extends Processor {
     @Override
     public void process() {
         for (SDField field : search.allFieldsList()) {
-            if ( field.getDataType() instanceof TensorDataType ) {
+            if (field.getDataType() == DataType.TENSOR) {
                 warnUseOfTensorFieldAsAttribute(field);
                 validateIndexingScripsForTensorField(field);
                 validateAttributeSettingForTensorField(field);
-            }
-            else if (field.getDataType() instanceof CollectionDataType){
-                validateDataTypeForCollectionField(field);
+            } else {
+                validateDataTypeForField(field);
             }
         }
     }
@@ -58,9 +55,9 @@ public class TensorFieldProcessor extends Processor {
         }
     }
 
-    private void validateDataTypeForCollectionField(SDField field) {
-        if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType)
+    private void validateDataTypeForField(SDField field) {
+        if (field.getDataType().getPrimitiveType() == DataType.TENSOR) {
             fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
+        }
     }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
index 9ee5f48a906..1349abc3795 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -2,7 +2,6 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.document.TensorDataType;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.document.DataType;
@@ -50,25 +49,22 @@ public class ValidateFieldTypes extends Processor {
         DataType seenType = seenFields.get(fieldName);
         if (seenType == null) {
             seenFields.put(fieldName, fieldType);
-        } else if ( ! compatibleTypes(seenType, fieldType)) {
-            throw newProcessException(searchName, fieldName, "Incompatible types. Expected " + 
+        } else if ( ! equalTypes(seenType, fieldType)) {
+            throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + 
                                                              seenType.getName() + " for " + fieldDesc +
                                                              " '" + fieldName + "', got " + fieldType.getName() + ".");
         }
     }
 
-    private boolean compatibleTypes(DataType seenType, DataType fieldType) {
+    private boolean equalTypes(DataType d1, DataType d2) {
         // legacy tag field type compatibility; probably not needed any more (Oct 2016)
-        if ("tag".equals(seenType.getName())) {
-            return "tag".equals(fieldType.getName()) || "WeightedSet<string>".equals(fieldType.getName());
+        if ("tag".equals(d1.getName())) {
+            return "tag".equals(d2.getName()) || "WeightedSet<string>".equals(d2.getName());
         }
-        if ("tag".equals(fieldType.getName())) {
-            return "tag".equals(seenType.getName()) || "WeightedSet<string>".equals(seenType.getName());
+        if ("tag".equals(d2.getName())) {
+            return "tag".equals(d1.getName()) || "WeightedSet<string>".equals(d1.getName());
         }
-        if (seenType instanceof TensorDataType && fieldType instanceof TensorDataType) {
-            return fieldType.isAssignableFrom(seenType); // TODO: Just do this for all types
-        }
-        return seenType.equals(fieldType);
+        return d1.equals(d2);
     }
     
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 705ec4dc4a1..3cdcf55b34e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -17,39 +17,39 @@ import java.util.Set;
 
 /**
  * @author    baldersheim
+ * @since     2010-02-19
  */
 public class DocumentManager {
 
-    public DocumentmanagerConfig.Builder produce(DocumentModel model, 
-                                                 DocumentmanagerConfig.Builder documentConfigBuilder) {
-        documentConfigBuilder.enablecompression(false);
+    public DocumentmanagerConfig.Builder produce(DocumentModel model, DocumentmanagerConfig.Builder docman) {
+        docman.enablecompression(false);
         Set<DataType> handled = new HashSet<>();
         for(NewDocumentType documentType : model.getDocumentManager().getTypes()) {
-            buildConfig(documentType, documentConfigBuilder, handled);
-            buildConfig(documentType.getAnnotations(), documentConfigBuilder);
+            handle(documentType, docman, handled);
+            handleAnnotations(documentType.getAnnotations(), docman);
             if ( documentType != VespaDocumentType.INSTANCE) {
-                DocumentmanagerConfig.Datatype.Builder dataTypeBuilder = new DocumentmanagerConfig.Datatype.Builder();
-                documentConfigBuilder.datatype(dataTypeBuilder);
-                buildConfig(documentType, dataTypeBuilder);
+                DocumentmanagerConfig.Datatype.Builder dt = new DocumentmanagerConfig.Datatype.Builder();
+                docman.datatype(dt);
+                handleDataType(documentType, dt);
             }
         }
-        return documentConfigBuilder;
+        return docman;
     }
 
-    private void buildConfig(DataTypeCollection type, DocumentmanagerConfig.Builder documentConfigBuilder, Set<DataType> built) {
+    private void handle(DataTypeCollection type, DocumentmanagerConfig.Builder docman, Set<DataType> handled) {
         for (DataType dataType : type.getTypes()) {
-            if (built.contains(dataType)) continue;
-            built.add(dataType);
+            if (handled.contains(dataType)) continue;
+            handled.add(dataType);
             if (dataType instanceof TemporaryStructuredDataType) continue;
-            if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) {
-                Datatype.Builder dataTypeBuilder = new Datatype.Builder();
-                documentConfigBuilder.datatype(dataTypeBuilder);
-                buildConfig(dataType, dataTypeBuilder);
+            if ((dataType.getId() < 0) || (DataType.lastPredefinedDataTypeId() < dataType.getId())) {
+                Datatype.Builder dtc = new Datatype.Builder();
+                docman.datatype(dtc);
+                handleDataType(dataType, dtc);
             }
         }
     }
 
-    private void buildConfig(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) {
+    private void handleAnnotation(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) {
         atb.
             id(type.getId()).
             name(type.getName());
@@ -62,34 +62,34 @@ public class DocumentManager {
             }
         }
     }
-    private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
+    private void handleAnnotations(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
         for (AnnotationType type : types) {
             DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
-            buildConfig(type, atb);
+            handleAnnotation(type, atb);
             builder.annotationtype(atb);
         }
     }
 
-    private void buildConfig(DataType type, Datatype.Builder builder) {
-        builder.id(type.getId());
+    private void handleDataType(DataType type, Datatype.Builder dtc) {
+        dtc.id(type.getId());
         if (type instanceof ArrayDataType) {
             CollectionDataType dt = (CollectionDataType) type;
-            builder.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId()));
+            dtc.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId()));
         } else if (type instanceof WeightedSetDataType) {
             WeightedSetDataType dt = (WeightedSetDataType) type;
-            builder.weightedsettype(new Datatype.Weightedsettype.Builder().
+            dtc.weightedsettype(new Datatype.Weightedsettype.Builder().
                     datatype(dt.getNestedType().getId()).
                     createifnonexistant(dt.createIfNonExistent()).
                     removeifzero(dt.removeIfZero()));
         } else if (type instanceof MapDataType) {
             MapDataType mtype = (MapDataType) type;
-            builder.maptype(new Datatype.Maptype.Builder().
+            dtc.maptype(new Datatype.Maptype.Builder().
                     keytype(mtype.getKeyType().getId()).
                     valtype(mtype.getValueType().getId()));
         } else if (type instanceof DocumentType) {
             DocumentType dt = (DocumentType) type;
             Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
-            builder.documenttype(doc);
+            dtc.documenttype(doc);
             doc.
                 name(dt.getName()).
                 headerstruct(dt.getHeaderType().getId()).
@@ -100,7 +100,7 @@ public class DocumentManager {
         } else if (type instanceof NewDocumentType) {
             NewDocumentType dt = (NewDocumentType) type;
             Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
-            builder.documenttype(doc);
+            dtc.documenttype(doc);
             doc.
                 name(dt.getName()).
                 headerstruct(dt.getHeader().getId()).
@@ -108,56 +108,49 @@ public class DocumentManager {
             for (NewDocumentType inherited : dt.getInherited()) {
                 doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
             }
-            buildConfig(dt.getFieldSets(), doc);
+            handleFieldSets(dt.getFieldSets(), doc);
         } else if (type instanceof TemporaryStructuredDataType) {
             //Ignored
         } else if (type instanceof StructDataType) {
-            StructDataType structType = (StructDataType) type;
-            Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder();
-            builder.structtype(structBuilder);
-            structBuilder.name(structType.getName());
-            if (structType.getCompressionConfig().type.getCode() != 0) {
-                structBuilder.
-                    compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(structType.getCompressionConfig().type.toString())).
-                    compresslevel(structType.getCompressionConfig().compressionLevel).
-                    compressthreshold((int)structType.getCompressionConfig().threshold).
-                    compressminsize((int)structType.getCompressionConfig().minsize);
+            StructDataType dt = (StructDataType) type;
+            Datatype.Structtype.Builder st = new Datatype.Structtype.Builder();
+            dtc.structtype(st);
+            st.name(dt.getName());
+            if (dt.getCompressionConfig().type.getCode() != 0) {
+                st.
+                    compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(dt.getCompressionConfig().type.toString())).
+                    compresslevel(dt.getCompressionConfig().compressionLevel).
+                    compressthreshold((int)dt.getCompressionConfig().threshold).
+                    compressminsize((int)dt.getCompressionConfig().minsize);
             }
-            for (com.yahoo.document.Field field : structType.getFieldsThisTypeOnly()) {
-                Datatype.Structtype.Field.Builder fieldBuilder = new Datatype.Structtype.Field.Builder();
-                structBuilder.field(fieldBuilder);
-                fieldBuilder.name(field.getName());
+            for (com.yahoo.document.Field field : dt.getFieldsThisTypeOnly()) {
+                Datatype.Structtype.Field.Builder fb = new Datatype.Structtype.Field.Builder();
+                st.field(fb);
+                fb.name(field.getName());
                 if (field.hasForcedId()) {
-                    fieldBuilder.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId()));
+                    fb.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId()));
                 }
-                fieldBuilder.datatype(field.getDataType().getId());
-
-                if (field.getDataType() instanceof TensorDataType)
-                    fieldBuilder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString());
+                fb.datatype(field.getDataType().getId());
             }
-            for (StructDataType inherited : structType.getInheritedTypes()) {
-                structBuilder.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName()));
+            for (StructDataType inherited : dt.getInheritedTypes()) {
+                st.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName()));
             }
         } else if (type instanceof AnnotationReferenceDataType) {
             AnnotationReferenceDataType annotationRef = (AnnotationReferenceDataType) type;
-            builder.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName()));
-        } else if (type instanceof TensorDataType) {
-            // Nothing to do; the type of the tensor is instead stored in each field as detailed type information
-            // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
-            // without changing the field type and thus requiring data refeed
+            dtc.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName()));
         } else {
-            throw new IllegalArgumentException("Can not create config for data type '" + type.getName());
+            throw new IllegalArgumentException("Can not handle datatype '" + type.getName());
         }
     }
 
-    private void buildConfig(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) {
+    private void handleFieldSets(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) {
+
         for (FieldSet builtinFs : fieldSets) {
-            buildConfig(builtinFs, doc);
+            handleFieldSet(builtinFs, doc);
         }
     }
 
-    private void buildConfig(FieldSet fs, Datatype.Documenttype.Builder doc) {
+    private void handleFieldSet(FieldSet fs, Datatype.Documenttype.Builder doc) {
         doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index ebe18b36410..2ba34def464 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -2,11 +2,13 @@
 package com.yahoo.vespa.configmodel.producers;
 
 import com.yahoo.document.*;
+import com.yahoo.document.DocumenttypesConfig.Builder;
 import com.yahoo.document.annotation.AnnotationReferenceDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.documentmodel.DataTypeCollection;
 import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.FieldSets;
 import com.yahoo.searchdefinition.document.FieldSet;
 import com.yahoo.vespa.documentmodel.DocumentModel;
 import java.util.*;
@@ -29,12 +31,12 @@ public class DocumentTypes {
             for (NewDocumentType inherited : documentType.getInherited()) {
                 produceInheritOrder(inherited, builder, produced);
             }
-            buildConfig(documentType, builder);
+            handle(documentType, builder);
             produced.put(documentType.getFullName(), documentType);
         }
     }
 
-    private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
+    private void handle(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
         if (documentType == VespaDocumentType.INSTANCE) {
             return;
         }
@@ -44,40 +46,40 @@ public class DocumentTypes {
             name(documentType.getName()).
             headerstruct(documentType.getHeader().getId()).
             bodystruct(documentType.getBody().getId());
-        Set<Integer> built = new HashSet<>();
+        Set<Integer> handled = new HashSet<>();
         for (NewDocumentType inherited : documentType.getInherited()) {
             db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId()));
-            markAsBuilt(built, inherited.getAllTypes());
+            markAsHandled(handled, inherited.getAllTypes());
         }
         for (DataType dt : documentType.getTypes()) {
-            buildConfig(dt, db, built);
+            handle(dt, db, handled);
         }
         for(AnnotationType annotation : documentType.getAnnotations()) {
             DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder();
             db.annotationtype(atb);
-            buildConfig(annotation, atb);
+            handle(annotation, atb);
         }
-        buildConfig(documentType.getFieldSets(), db);
+        handleFieldSets(documentType.getFieldSets(), db);
         builder.documenttype(db);
     }
 
-    private void buildConfig(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) {
+    private void handleFieldSets(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) {
         for (FieldSet fs : fieldSets) {
-            buildConfig(fs, db);
+            handleFieldSet(fs, db);
         }        
     }
 
-    private void buildConfig(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
+    private void handleFieldSet(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
         db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
 
-    private void markAsBuilt(Set<Integer> built, DataTypeCollection typeCollection) {
+    private void markAsHandled(Set<Integer> handled, DataTypeCollection typeCollection) {
         for (DataType type : typeCollection.getTypes()) {
-            built.add(type.getId());
+            handled.add(type.getId());
         }
     }
 
-    private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
+    private void handle(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
         builder.
             id(annotation.getId()).
             name(annotation.getName());
@@ -90,77 +92,67 @@ public class DocumentTypes {
         }
     }
 
-    private void buildConfig(DataType type, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) {
-        if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! built.contains(type.getId())) {
-            built.add(type.getId());
-            DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder();
-            dataTypeBuilder.id(type.getId());
+    private void handle(DataType type, DocumenttypesConfig.Documenttype.Builder db, Set<Integer> handled) {
+        if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! handled.contains(type.getId())) {
+            handled.add(type.getId());
+            DocumenttypesConfig.Documenttype.Datatype.Builder dtb = new DocumenttypesConfig.Documenttype.Datatype.Builder();
+            dtb.id(type.getId());
             if (type instanceof StructDataType) {
-                dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT);
+                dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("STRUCT"));
                 StructDataType dt = (StructDataType) type;
-                DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
-                dataTypeBuilder.sstruct(structBuilder);
-                structBuilder.name(dt.getName());
+                DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder sb = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
+                dtb.sstruct(sb);
+                sb.name(dt.getName());
                 if (dt.getCompressionConfig().type.getCode() != 0) {
-                    structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder().
+                    sb.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder().
                             type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(dt.getCompressionConfig().type.toString())).
                             level(dt.getCompressionConfig().compressionLevel).
                             threshold((int)dt.getCompressionConfig().threshold).
                             minsize((int)dt.getCompressionConfig().minsize));
                 }
                 for (com.yahoo.document.Field field : dt.getFields()) {
-                    DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder =
-                            new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder();
-                    builder.name(field.getName()).
+                    sb.field(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder().
+                            name(field.getName()).
                             id(field.getId()).
                             id_v6(field.getIdV6()).
-                            datatype(field.getDataType().getId());
-                    if (field.getDataType() instanceof TensorDataType)
-                        builder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString());
-                    structBuilder.field(builder);
-                    buildConfig(field.getDataType(), documentBuilder, built);
+                            datatype(field.getDataType().getId()));
+                    handle(field.getDataType(), db, handled);
                 }
             } else if (type instanceof ArrayDataType) {
-                dataTypeBuilder.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY).
+                dtb.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ARRAY")).
                     array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder().
                             element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(((ArrayDataType)type).getNestedType().getId())));
-                buildConfig(((ArrayDataType)type).getNestedType(), documentBuilder, built);
+                handle(((ArrayDataType)type).getNestedType(), db, handled);
             } else if (type instanceof WeightedSetDataType) {
-                dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET).
+                dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("WSET")).
                 wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder().
                         key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder().
                                 id(((WeightedSetDataType)type).getNestedType().getId())).
                         createifnonexistent(((WeightedSetDataType)type).createIfNonExistent()).
                         removeifzero(((WeightedSetDataType)type).removeIfZero()));
-                buildConfig(((WeightedSetDataType)type).getNestedType(), documentBuilder, built);
+                handle(((WeightedSetDataType)type).getNestedType(), db, handled);
             } else if (type instanceof MapDataType) {
-                dataTypeBuilder.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP).
+                dtb.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("MAP")).
                     map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder().
                         key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder().
                             id(((MapDataType)type).getKeyType().getId())).
                         value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder().
                             id(((MapDataType)type).getValueType().getId())));
-                buildConfig(((MapDataType)type).getKeyType(), documentBuilder, built);
-                buildConfig(((MapDataType)type).getValueType(), documentBuilder, built);
+                handle(((MapDataType)type).getKeyType(), db, handled);
+                handle(((MapDataType)type).getValueType(), db, handled);
             } else if (type instanceof AnnotationReferenceDataType) {
-                dataTypeBuilder.
-                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ANNOTATIONREF).
+                dtb.
+                    type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ANNOTATIONREF")).
                     annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder().
                             annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder().
                                     id(((AnnotationReferenceDataType)type).getAnnotationType().getId())));
-            } else if (type instanceof TensorDataType) {
-                // The type of the tensor is not stored here but instead in each field as detailed type information
-                // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
-                // without changing the field type and thus requiring data refeed
-                return;
             } else {
                 return;
             }
-            documentBuilder.datatype(dataTypeBuilder);
+            db.datatype(dtb);
         }
     }
-
 }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index 1d39d1e6928..c03fb0617b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -43,7 +43,7 @@ public class SearchDataTypeValidator extends Validator {
         for (Field field : doc.fieldSet()) {
             DataType fieldType = field.getDataType();
             disallowIndexingOfMaps(cluster, def, field);
-            if ( ! isSupportedInSearchClusters(fieldType)) {
+            if (!validateDataType(fieldType)) {
                 throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " +
                                                    "clusters (field '" + field.getName() + "' in definition '" +
                                                    def.getName() + "' for cluster '" + cluster.getClusterName() + "').");
@@ -51,30 +51,28 @@ public class SearchDataTypeValidator extends Validator {
         }
     }
 
-    private boolean isSupportedInSearchClusters(DataType dataType) {
-        if (dataType instanceof ArrayDataType || dataType instanceof WeightedSetDataType) {
-            return isSupportedInSearchClusters(((CollectionDataType)dataType).getNestedType());
+    private boolean validateDataType(DataType dataType) {
+        if (dataType instanceof ArrayDataType ||
+            dataType instanceof WeightedSetDataType)
+        {
+            return validateDataType(((CollectionDataType)dataType).getNestedType());
         }
-        else if (dataType instanceof StructDataType) {
+        if (dataType instanceof StructDataType) {
             return true; // Struct will work for summary TODO maybe check individual fields
         }
-        else if (dataType instanceof MapDataType) {
+        if (dataType instanceof MapDataType) {
             return true; // Maps will work for summary, see disallowIndexingOfMaps()
         }
-        else if (dataType instanceof TensorDataType) {
-            return true;
-        }
-        else {
-            return dataType.equals(DataType.INT) ||
-                   dataType.equals(DataType.FLOAT) ||
-                   dataType.equals(DataType.STRING) ||
-                   dataType.equals(DataType.RAW) ||
-                   dataType.equals(DataType.LONG) ||
-                   dataType.equals(DataType.DOUBLE) ||
-                   dataType.equals(DataType.URI) ||
-                   dataType.equals(DataType.BYTE) ||
-                   dataType.equals(DataType.PREDICATE);
-        }
+        return dataType.equals(DataType.INT) ||
+               dataType.equals(DataType.FLOAT) ||
+               dataType.equals(DataType.STRING) ||
+               dataType.equals(DataType.RAW) ||
+               dataType.equals(DataType.LONG) ||
+               dataType.equals(DataType.DOUBLE) ||
+               dataType.equals(DataType.URI) ||
+               dataType.equals(DataType.BYTE) ||
+               dataType.equals(DataType.PREDICATE) ||
+               dataType.equals(DataType.TENSOR);
     }
 
     private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 3fc84f2860b..32be8906ffc 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -805,14 +805,13 @@ DataType dataType() :
     DataType mapType=null;
 	DataType arrayType=null;
     DataType wsetType=null;
-    TensorType tensorType;
+
 }
 {
     (   LOOKAHEAD(<ARRAY> <LESSTHAN>)       ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } )
       | LOOKAHEAD(<WEIGHTEDSET> <LESSTHAN>) ( <WEIGHTEDSET> <LESSTHAN> wsetType = dataType() <GREATERTHAN> { return  DataType.getWeightedSet(wsetType); } )
       | LOOKAHEAD(<MAP> <LESSTHAN>)         ( mapType = mapDataType() { return mapType; } )
       | LOOKAHEAD(<ANNOTATIONREFERENCE> <LESSTHAN>) ( mapType = annotationRefDataType() { return mapType; } )
-      | LOOKAHEAD(<TENSOR_TYPE>)                 ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } )
       | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] )
     )
     {