Maintain TensorType in documents

4 files changed, 37 insertions, 56 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
index 793a5fcff6c..09aa55f776b 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
@@ -32,7 +32,6 @@ public class VespaDocumentType {
         vespa.add(PositionDataType.INSTANCE);
         vespa.add(DataType.URI);
         vespa.add(DataType.PREDICATE);
-        vespa.add(DataType.TENSOR);
         return vespa;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index 950ec791368..5856caeb692 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -25,10 +25,6 @@ public final class Attribute implements Cloneable, Serializable {
     private Type type;
     private CollectionType collectionType;
 
-    /** True if only the enum information should be read from this attribute
-     *  (i.e. the actual values are not relevant, only which documents have the
-     *   same values) Used for collapsing and unique.
-     */
     private boolean removeIfZero = false;
     private boolean createIfNonExistent = false;
     private boolean enableBitVectors = false;
@@ -224,31 +220,22 @@ public final class Attribute implements Cloneable, Serializable {
     }
 
     /** Converts to the right field type from an attribute type */
-    public static DataType convertAttrType(Type attrType) {
-        if (attrType== Type.STRING) {
-            return DataType.STRING;
-        } else if (attrType== Type.INTEGER) {
-            return DataType.INT;
-        } else if (attrType== Type.LONG) {
-            return DataType.LONG;
-        } else if (attrType== Type.FLOAT) {
-            return DataType.FLOAT;
-        } else if (attrType== Type.DOUBLE) {
-            return DataType.DOUBLE;
-        } else if (attrType == Type.BYTE) {
-            return DataType.BYTE;
-        } else if (attrType == Type.PREDICATE) {
-            return DataType.PREDICATE;
-        } else if (attrType == Type.TENSOR) {
-            return DataType.TENSOR;
-        } else {
-            throw new IllegalArgumentException("Don't know which attribute type to " +
-                    "convert " + attrType + " to");
+    private DataType toDataType(Type attributeType) {
+        switch (attributeType) {
+            case STRING : return DataType.STRING;
+            case INTEGER: return DataType.INT;
+            case LONG: return DataType.LONG;
+            case FLOAT: return DataType.FLOAT;
+            case DOUBLE: return DataType.DOUBLE;
+            case BYTE: return DataType.BYTE;
+            case PREDICATE: return DataType.PREDICATE;
+            case TENSOR: DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new));
+            default: throw new IllegalArgumentException("Unknown attribute type " + attributeType);
         }
     }
 
     public DataType getDataType() {
-        DataType dataType = Attribute.convertAttrType(type);
+        DataType dataType = toDataType(type);
         if (collectionType.equals(Attribute.CollectionType.ARRAY)) {
             return DataType.getArray(dataType);
         } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index ae16f6cfed8..f618c55adfc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -23,13 +23,11 @@ public class TensorFieldProcessor extends Processor {
     @Override
     public void process() {
         for (SDField field : search.allFieldsList()) {
-            if (field.getDataType() == DataType.TENSOR) {
-                warnUseOfTensorFieldAsAttribute(field);
-                validateIndexingScripsForTensorField(field);
-                validateAttributeSettingForTensorField(field);
-            } else {
-                validateDataTypeForField(field);
-            }
+            if ( ! (field.getDataType() instanceof TensorDataType)) continue;
+
+            warnUseOfTensorFieldAsAttribute(field);
+            validateIndexingScripsForTensorField(field);
+            validateAttributeSettingForTensorField(field);
         }
     }
 
@@ -55,9 +53,4 @@ public class TensorFieldProcessor extends Processor {
         }
     }
 
-    private void validateDataTypeForField(SDField field) {
-        if (field.getDataType().getPrimitiveType() == DataType.TENSOR) {
-            fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
-        }
-    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index c03fb0617b8..1d39d1e6928 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -43,7 +43,7 @@ public class SearchDataTypeValidator extends Validator {
         for (Field field : doc.fieldSet()) {
             DataType fieldType = field.getDataType();
             disallowIndexingOfMaps(cluster, def, field);
-            if (!validateDataType(fieldType)) {
+            if ( ! isSupportedInSearchClusters(fieldType)) {
                 throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " +
                                                    "clusters (field '" + field.getName() + "' in definition '" +
                                                    def.getName() + "' for cluster '" + cluster.getClusterName() + "').");
@@ -51,28 +51,30 @@ public class SearchDataTypeValidator extends Validator {
         }
     }
 
-    private boolean validateDataType(DataType dataType) {
-        if (dataType instanceof ArrayDataType ||
-            dataType instanceof WeightedSetDataType)
-        {
-            return validateDataType(((CollectionDataType)dataType).getNestedType());
+    private boolean isSupportedInSearchClusters(DataType dataType) {
+        if (dataType instanceof ArrayDataType || dataType instanceof WeightedSetDataType) {
+            return isSupportedInSearchClusters(((CollectionDataType)dataType).getNestedType());
         }
-        if (dataType instanceof StructDataType) {
+        else if (dataType instanceof StructDataType) {
             return true; // Struct will work for summary TODO maybe check individual fields
         }
-        if (dataType instanceof MapDataType) {
+        else if (dataType instanceof MapDataType) {
             return true; // Maps will work for summary, see disallowIndexingOfMaps()
         }
-        return dataType.equals(DataType.INT) ||
-               dataType.equals(DataType.FLOAT) ||
-               dataType.equals(DataType.STRING) ||
-               dataType.equals(DataType.RAW) ||
-               dataType.equals(DataType.LONG) ||
-               dataType.equals(DataType.DOUBLE) ||
-               dataType.equals(DataType.URI) ||
-               dataType.equals(DataType.BYTE) ||
-               dataType.equals(DataType.PREDICATE) ||
-               dataType.equals(DataType.TENSOR);
+        else if (dataType instanceof TensorDataType) {
+            return true;
+        }
+        else {
+            return dataType.equals(DataType.INT) ||
+                   dataType.equals(DataType.FLOAT) ||
+                   dataType.equals(DataType.STRING) ||
+                   dataType.equals(DataType.RAW) ||
+                   dataType.equals(DataType.LONG) ||
+                   dataType.equals(DataType.DOUBLE) ||
+                   dataType.equals(DataType.URI) ||
+                   dataType.equals(DataType.BYTE) ||
+                   dataType.equals(DataType.PREDICATE);
+        }
     }
 
     private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {