diff options
Diffstat (limited to 'document/src')
32 files changed, 348 insertions, 248 deletions
diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java index 640bd94bd1c..4b24bb1ae00 100644 --- a/document/src/main/java/com/yahoo/document/ArrayDataType.java +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -8,9 +8,10 @@ import java.util.ArrayList; import java.util.List; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class ArrayDataType extends CollectionDataType { + // The global class identifier shared with C++. public static int classId = registerClass(Ids.document + 54, ArrayDataType.class); diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 51af799efd9..184340f295e 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -14,6 +14,7 @@ import com.yahoo.document.datatypes.Raw; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.UriFieldValue; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Ids; import com.yahoo.vespa.objects.ObjectVisitor; @@ -46,40 +47,30 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory()); public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory()); public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory()); - // ARRAY is type 6, but never used, array IDs are generated - // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class); public final static DocumentType DOCUMENT = new DocumentType("document"); - // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); - // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class); - // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); - // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated - // Tags are converted to weightedset<string> when reading the search definition - public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); - // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class); - // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now.. - // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); - public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory()); + public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately + // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor + + // Tags are converted to weightedset<string> when reading the search definition TODO: Remove it + public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); public static int lastPredefinedDataTypeId() { return 21; } - /** - * Set to true when this type is registered in a type manager. From that time we should refuse changes. - */ + /** Set to true when this type is registered in a type manager. From that time we should refuse changes. */ private boolean registered = false; private String name; - /** - * The id of this type - */ + /** The id of this type */ private int dataTypeId; static final private CopyOnWriteHashMap<Pair, Constructor> constructorCache = new CopyOnWriteHashMap<>(); + /** * Creates a datatype * @@ -96,26 +87,21 @@ public abstract class DataType extends Identifiable implements Serializable, Com return (DataType)super.clone(); } - public void setRegistered() { + void setRegistered() { registered = true; } - public boolean isRegistered() { return registered; } - /** - * Creates a new, empty FieldValue of this type. - * - * @return a new, empty FieldValue of this type. - */ + /** Creates a new, empty FieldValue of this type */ public abstract FieldValue createFieldValue(); /** - * This will try to create the object by reflection. This can be very expensive - * so some might discourage that. - * @param arg The constructor argument. - * @return Fully constructed value. + * Creates a field value by reflection + * + * @param arg the value of the newly created field value + * @return a fully constructed value */ protected FieldValue createByReflection(Object arg) { Class<?> valClass = getValueClass(); @@ -141,8 +127,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com /** * Creates a new FieldValue of this type, with the given value. * - * @param arg the value that the new FieldValue shall have. - * @return A new FieldValue of this type, with the given value. + * @param arg the value of the new FieldValue + * @return a new FieldValue of this type, with the given value */ public FieldValue createFieldValue(Object arg) { if (arg == null) { @@ -211,12 +197,17 @@ public abstract class DataType extends Identifiable implements Serializable, Com return new WeightedSetDataType(type, createIfNonExistent, removeIfZero); } + /** Returns the given tensor type as a DataType */ + public static TensorDataType getTensor(TensorType type) { + return new TensorDataType(type); + } + public String getName() { return name; } /** - * Sets the name of this data type. WARNING! Do not use! + * Sets the name of this data type. WARNING! Do not use! * * @param name the name of this data type. */ @@ -229,7 +220,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Sets the ID of this data type. WARNING! Do not use! + * Sets the ID of this data type. WARNING! Do not use! * * @param id the ID of this data type. */ @@ -238,7 +229,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Registeres this type in the given document manager. + * Registers this type in the given document manager. * * @param manager the DocumentTypeManager to register in. */ @@ -255,14 +246,12 @@ public abstract class DataType extends Identifiable implements Serializable, Com } public boolean equals(Object other) { - if (!(other instanceof DataType)) { - return false; - } + if (!(other instanceof DataType)) return false; DataType type = (DataType)other; return (name.equals(type.name) && dataTypeId == type.dataTypeId); } - public java.lang.String toString() { + public String toString() { return "datatype " + name + " (code: " + dataTypeId + ")"; } @@ -278,8 +267,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com */ public FieldPath buildFieldPath(String fieldPathString) { if (fieldPathString.length() > 0) { - throw new IllegalArgumentException( - "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString); + throw new IllegalArgumentException("Datatype " + toString() + + " does not support further recursive structure: " + fieldPathString); } return new FieldPath(); } diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 34c952e1cec..e1d912b4e51 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -26,7 +26,7 @@ import java.util.Map; * be removed soon. * * @author bratseth - * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class Document extends StructuredFieldValue { @@ -267,7 +267,9 @@ public class Document extends StructuredFieldValue { } /** Returns true if the argument is a document which has the same set of values */ + @Override public boolean equals(Object o) { + if (o == this) return true; if (!(o instanceof Document)) return false; Document other = (Document) o; return (super.equals(o) && docId.equals(other.docId) && @@ -394,4 +396,5 @@ public class Document extends StructuredFieldValue { comp = body.compareTo(otherValue.body); return comp; } + } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index 0de7fe60500..1b3370cfa66 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -12,6 +12,7 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.VespaDocumentDeserializer42; import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.TensorType; import java.lang.reflect.Modifier; import java.util.*; @@ -28,13 +29,23 @@ import java.util.logging.Logger; * datatypes, for instance displaying the data type in human-readable form * or as XML. * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public class DocumentTypeManager { private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); private ConfigSubscriber subscriber; + // *Configured data types* (not built-in/primitive) indexed by their id + // + // *Primitive* data types are always available and have a single id. + // + // *Built-in dynamic* types: The tensor type. + // Any tensor type has the same id and is always available just like primitive types. + // However, unlike primitive types, each tensor type is a separate DataType instance + // (which carries additional type information (detailedType) supplied at runtime). + // The reason for this is that we want the data type to stay the same when we change the detailed tensor type + // to maintain compatibility on (some) tensor type changes. private Map<Integer, DataType> dataTypes = new LinkedHashMap<>(); private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>(); private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry(); @@ -85,6 +96,7 @@ public class DocumentTypeManager { } public boolean hasDataType(String name) { + if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { return true; @@ -94,10 +106,14 @@ public class DocumentTypeManager { } public boolean hasDataType(int code) { + if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present return dataTypes.containsKey(code); } public DataType getDataType(String name) { + if (name.startsWith("tensor(")) // built-in dynamic + return new TensorDataType(TensorType.fromSpec(name)); + List<DataType> foundTypes = new ArrayList<>(); for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { @@ -125,7 +141,19 @@ public class DocumentTypeManager { return foundTypes.get(0); } - public DataType getDataType(int code) { + public DataType getDataType(int code) { return getDataType(code, ""); } + + /** + * Return a data type instance + * + * @param code the code of the data type to return, which must be either built in or present in this manager + * @param detailedType detailed type information, or the empty string if none + * @return the appropriate DataType instance + */ + public DataType getDataType(int code, String detailedType) { + if (code == DataType.tensorDataTypeCode) // built-in dynamic + return new TensorDataType(TensorType.fromSpec(detailedType)); + DataType type = dataTypes.get(code); if (type == null) { StringBuilder types=new StringBuilder(); @@ -138,11 +166,11 @@ public class DocumentTypeManager { } } - DataType getDataTypeAndReturnTemporary(int code) { + DataType getDataTypeAndReturnTemporary(int code, String detailedType) { if (hasDataType(code)) { - return getDataType(code); + return getDataType(code, detailedType); } - return new TemporaryDataType(code); + return new TemporaryDataType(code, detailedType); } /** @@ -156,9 +184,11 @@ public class DocumentTypeManager { /** * Register a single datatype. Re-registering an existing, but equal, datatype is ok. + * * @param type The datatype to register */ void registerSingleType(DataType type) { + if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly if (dataTypes.containsKey(type.getId())) { DataType existingType = dataTypes.get(type.getId()); if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType)) @@ -188,7 +218,7 @@ public class DocumentTypeManager { dataTypes.remove(existingType.getId()); } else { throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered " - + type + ", but already uses id " + type.getId()); + + type + ", but already uses id " + type.getId()); } } @@ -250,7 +280,8 @@ public class DocumentTypeManager { } /** - * A read only view of the registered data types + * Returns a read only view of the registered data types + * * @return collection of types */ public Collection<DataType> getDataTypes() { @@ -318,7 +349,7 @@ public class DocumentTypeManager { for (Field field : structDataType.getFieldsThisTypeOnly()) { DataType fieldType = field.getDataType(); if (fieldType instanceof TemporaryDataType) { - field.setDataType(getDataType(fieldType.getCode())); + field.setDataType(getDataType(fieldType.getCode(), ((TemporaryDataType)fieldType).getDetailedType())); } else { if (!seenStructs.contains(fieldType)) { replaceTemporaryTypes(fieldType, seenStructs); @@ -329,7 +360,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List<DataType> seenStructs) { if (collectionDataType.getNestedType() instanceof TemporaryDataType) { - collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode())); + collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode(), "")); } else { replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs); } @@ -337,13 +368,13 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInMap(MapDataType mapDataType, List<DataType> seenStructs) { if (mapDataType.getValueType() instanceof TemporaryDataType) { - mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode())); + mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode(), "")); } else { replaceTemporaryTypes(mapDataType.getValueType(), seenStructs); } if (mapDataType.getKeyType() instanceof TemporaryDataType) { - mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode())); + mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode(), "")); } else { replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs); } @@ -351,7 +382,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List<DataType> seenStructs) { if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) { - weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode())); + weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode(), "")); } else { replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs); } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 9d7139e6226..7b5f1392032 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; /** * Configures the Vespa document manager from a config id. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{ @@ -82,21 +82,21 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub try { for (Object o : thisDataType.arraytype()) { DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o; - DataType nestedType = manager.getDataType(array.datatype()); + DataType nestedType = manager.getDataType(array.datatype(), ""); ArrayDataType type = new ArrayDataType(nestedType, id); manager.register(type); } for (Object o : thisDataType.maptype()) { DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o; - DataType keyType = manager.getDataType(map.keytype()); - DataType valType = manager.getDataType(map.valtype()); + DataType keyType = manager.getDataType(map.keytype(), ""); + DataType valType = manager.getDataType(map.valtype(), ""); MapDataType type = new MapDataType(keyType, valType, id); manager.register(type); } for (Object o : thisDataType.weightedsettype()) { DocumentmanagerConfig.Datatype.Weightedsettype wset = (DocumentmanagerConfig.Datatype.Weightedsettype) o; - DataType nestedType = manager.getDataType(wset.datatype()); + DataType nestedType = manager.getDataType(wset.datatype(), ""); WeightedSetDataType type = new WeightedSetDataType( nestedType, wset.createifnonexistant(), wset.removeifzero(), id); manager.register(type); @@ -114,8 +114,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentmanagerConfig.Datatype.Structtype.Field field = (DocumentmanagerConfig.Datatype.Structtype.Field) j; DataType fieldType = (field.datatype() == id) - ? manager.getDataTypeAndReturnTemporary(field.datatype()) - : manager.getDataType(field.datatype()); + ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype()) + : manager.getDataType(field.datatype(), field.detailedtype()); if (field.id().size() == 1) { type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); @@ -127,8 +127,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } for (Object o : thisDataType.documenttype()) { DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o; - StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct()); - StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct()); + StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); + StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), ""); for (Field field : body.getFields()) { field.setHeader(false); } @@ -210,7 +210,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) { for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id()); - DataType payload = manager.getDataType(annType.datatype()); + DataType payload = manager.getDataType(annType.datatype(), ""); if (!payload.equals(DataType.NONE)) { annotationType.setDataType(payload); } @@ -234,7 +234,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub int id = thisDataType.id(); for (Object o : thisDataType.structtype()) { DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; - StructDataType thisStruct = (StructDataType) manager.getDataType(id); + StructDataType thisStruct = (StructDataType) manager.getDataType(id, ""); for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 86543916b42..b80d92e0e4d 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -13,7 +13,7 @@ import java.io.Serializable; * A name and type. Fields are contained in document types to describe their fields, * but is also used to represent name/type pairs which are not part of document types. * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen * @author bratseth */ public class Field extends FieldBase implements FieldSet, Comparable, Serializable { diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java index a0024bb0497..23bf4b43ccf 100644 --- a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java +++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java @@ -8,9 +8,10 @@ import com.yahoo.vespa.objects.ObjectVisitor; import java.util.Objects; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class PrimitiveDataType extends DataType { + public static abstract class Factory { public abstract FieldValue create(); } diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java index da65dde72da..a0c6fcb889d 100644 --- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java +++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java @@ -4,11 +4,15 @@ package com.yahoo.document; import com.yahoo.document.datatypes.FieldValue; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ class TemporaryDataType extends DataType { - TemporaryDataType(int dataTypeId) { + + private final String detailedType; + + TemporaryDataType(int dataTypeId, String detailedType) { super("temporary_" + dataTypeId, dataTypeId); + this.detailedType = detailedType; } @Override @@ -25,4 +29,7 @@ class TemporaryDataType extends DataType { public boolean isValueCompatible(FieldValue value) { return false; } + + String getDetailedType() { return detailedType; } + } diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java new file mode 100644 index 00000000000..dbaf6ee7763 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/TensorDataType.java @@ -0,0 +1,50 @@ +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.objects.Ids; + +/** + * A DataType containing a tensor type + * + * @author bratseth + */ +public class TensorDataType extends DataType { + + private final TensorType tensorType; + + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 59, TensorDataType.class); + + public TensorDataType(TensorType tensorType) { + super(tensorType.toString(), DataType.tensorDataTypeCode); + this.tensorType = tensorType; + } + + public TensorDataType clone() { + return (TensorDataType)super.clone(); + } + + @Override + public FieldValue createFieldValue() { + return new TensorFieldValue(tensorType); + } + + @Override + public Class<? extends TensorFieldValue> getValueClass() { + return TensorFieldValue.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (value == null) return false; + if ( ! TensorFieldValue.class.isAssignableFrom(value.getClass())) return false; + TensorFieldValue tensorValue = (TensorFieldValue)value; + return tensorValue.getDataType().getTensorType().isAssignableTo(tensorType); + } + + /** Returns the type of the tensor this field can hold */ + public TensorType getTensorType() { return tensorType; } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 66cc472de69..09fb8b71db1 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -16,7 +16,7 @@ import java.util.*; /** * FieldValue which encapsulates a Array value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> implements List<T> { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index 5a01dc33aa1..b5920cb4758 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -215,7 +215,6 @@ public class Struct extends StructuredFieldValue { if (!super.equals(o)) return false; Struct struct = (Struct) o; - return values.equals(struct.values); } @@ -388,4 +387,5 @@ public class Struct extends StructuredFieldValue { } return fieldType.cast(fieldValue); } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java index b4585a2188d..7957d9b812f 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -10,7 +10,7 @@ import java.util.List; import java.util.Map; /** - * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + * @author Håkon Humberset */ public abstract class StructuredFieldValue extends CompositeFieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index bee29478219..9d8e9a83b5e 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -3,11 +3,12 @@ package com.yahoo.document.datatypes; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.serialization.FieldReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.document.serialization.XmlStream; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import java.util.Objects; import java.util.Optional; @@ -20,12 +21,18 @@ import java.util.Optional; public class TensorFieldValue extends FieldValue { private Optional<Tensor> tensor; + + private final TensorDataType dataType; - public TensorFieldValue() { - tensor = Optional.empty(); + /** Create an empty tensor field value */ + public TensorFieldValue(TensorType type) { + this.dataType = new TensorDataType(type); + this.tensor = Optional.empty(); } + /** Create a tensor field value containing the given tensor */ public TensorFieldValue(Tensor tensor) { + this.dataType = new TensorDataType(tensor.type()); this.tensor = Optional.of(tensor); } @@ -34,8 +41,8 @@ public class TensorFieldValue extends FieldValue { } @Override - public DataType getDataType() { - return DataType.TENSOR; + public TensorDataType getDataType() { + return dataType; } @Override @@ -51,16 +58,23 @@ public class TensorFieldValue extends FieldValue { @Override public void assign(Object o) { if (o == null) { - tensor = Optional.empty(); + assignTensor(Optional.empty()); } else if (o instanceof Tensor) { - tensor = Optional.of((Tensor)o); + assignTensor(Optional.of((Tensor)o)); } else if (o instanceof TensorFieldValue) { - tensor = ((TensorFieldValue)o).getTensor(); + assignTensor(((TensorFieldValue)o).getTensor()); } else { throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" + - o.getClass().getName() + "'."); + o.getClass().getName() + "'."); } } + + public void assignTensor(Optional<Tensor> tensor) { + if (tensor.isPresent() && ! dataType.getTensorType().isAssignableTo(tensor.get().type())) + throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() + + " to field of type " + dataType.getTensorType()); + this.tensor = tensor; + } @Override public void serialize(Field field, FieldWriter writer) { @@ -74,27 +88,14 @@ public class TensorFieldValue extends FieldValue { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TensorFieldValue)) { - return false; - } - TensorFieldValue rhs = (TensorFieldValue)o; - if (!Objects.equals(tensor, rhs.tensor)) { - return false; - } + if (this == o) return true; + if ( ! (o instanceof TensorFieldValue)) return false; + + TensorFieldValue other = (TensorFieldValue)o; + if ( ! dataType.getTensorType().equals(other.dataType.getTensorType())) return false; + if ( ! tensor.equals(other.tensor)) return false; return true; } - public static PrimitiveDataType.Factory getFactory() { - return new PrimitiveDataType.Factory() { - - @Override - public FieldValue create() { - return new TensorFieldValue(); - } - }; - } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index b1c635d7641..e0740338aae 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -583,71 +583,41 @@ public class JsonReader { } private void fillTensor(TensorFieldValue tensorFieldValue) { + Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); - Tensor.Builder tensorBuilder = null; // read tensor cell fields and ignore everything else for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { if (TENSOR_CELLS.equals(buffer.currentName())) - tensorBuilder = readTensorCells(tensorBuilder); + readTensorCells(tensorBuilder); } expectObjectEnd(buffer.currentToken()); - if (tensorBuilder == null) // no cells + no type: empty tensor type - tensorBuilder = Tensor.Builder.of(TensorType.empty); tensorFieldValue.assign(tensorBuilder.build()); } - private Tensor.Builder readTensorCells(Tensor.Builder tensorBuilder) { + private void readTensorCells(Tensor.Builder tensorBuilder) { expectArrayStart(buffer.currentToken()); int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - tensorBuilder = readTensorCell(tensorBuilder); - } + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + readTensorCell(tensorBuilder); expectCompositeEnd(buffer.currentToken()); - return tensorBuilder; } - private Tensor.Builder readTensorCell(Tensor.Builder tensorBuilder) { + private void readTensorCell(Tensor.Builder tensorBuilder) { expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = null; + Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { String currentName = buffer.currentName(); if (TENSOR_ADDRESS.equals(currentName)) { - if (tensorBuilder != null) { - cellBuilder = tensorBuilder.cell(); - readTensorAddress(cellBuilder); - } - else { // gnarly temporary path to create a type on the fly TODO; Remove when we always have a type - expectObjectStart(buffer.currentToken()); - int initNesting2 = buffer.nesting(); - List<Pair<String,String>> entries = new ArrayList<>(); - for (buffer.next(); buffer.nesting() >= initNesting2; buffer.next()) { - String dimension = buffer.currentName(); - String label = buffer.currentText(); - entries.add(new Pair<>(dimension, label)); - } - TensorType.Builder typeBuilder = new TensorType.Builder(); - for (Pair<String,String> entry : entries) - typeBuilder.mapped(entry.getFirst()); - tensorBuilder = Tensor.Builder.of(typeBuilder.build()); - cellBuilder = tensorBuilder.cell(); - for (Pair<String,String> entry : entries) - cellBuilder.label(entry.getFirst(), entry.getSecond()); - expectObjectEnd(buffer.currentToken()); - } + readTensorAddress(cellBuilder); } else if (TENSOR_VALUE.equals(currentName)) { cellValue = Double.valueOf(buffer.currentText()); } } expectObjectEnd(buffer.currentToken()); - if (tensorBuilder == null) { // no content TODO; This will go away with the above - tensorBuilder = Tensor.Builder.of(TensorType.empty); - cellBuilder = tensorBuilder.cell(); - } cellBuilder.value(cellValue); - return tensorBuilder; } private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) { diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java index 52a62caf296..5f1b227790b 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java @@ -9,7 +9,7 @@ import com.yahoo.document.DocumentTypeManager; /** * This interface is used to implement custom deserialization of document updates. * - * @author <a href="mailto:ravishar@yahoo-inc.com">Ravi Sharma</a> + * @author Ravi Sharma * @author baldersheim */ public interface DocumentReader { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 753008de7e0..6e9495b1437 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -115,8 +115,8 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple // Verify that we have correct version version = getShort(null); if (version < 6 || version > Document.SERIALIZED_VERSION) { - throw new DeserializationException( - "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + "."); + throw new DeserializationException("Unknown version " + version + ", expected " + + Document.SERIALIZED_VERSION + "."); } int dataLength = 0; @@ -278,7 +278,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); - value.assign(TypedBinaryFormat.decode(null, encodedTensor)); // TODO: Pass type + value.assign(TypedBinaryFormat.decode(value.getDataType().getTensorType(), encodedTensor)); } else { value.clear(); } @@ -328,7 +328,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); } - //save a reference to the big buffer we're reading from: + // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; if (version < 7) { diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index 413d1581e58..02773c7dad0 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import java.io.FileOutputStream; import java.io.IOException; @@ -43,6 +44,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { private final String documentId = "doc:something:foooo"; private final String tensorField = "tensorfield"; + private final TensorType tensorType = new TensorType.Builder().mapped("x").build(); private Document createDocument() { return new Document(docMan.getDocumentType("foobar"), new DocumentId(documentId)); @@ -60,7 +62,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { DataType stringwset = DataType.getWeightedSet(DataType.STRING); docType.addField(new Field("strwset", stringwset)); - docType.addField(new Field(tensorField, DataType.TENSOR)); + docType.addField(new Field(tensorField, new TensorDataType(tensorType))); docMan.register(docType); docType2 = new DocumentType("otherdoctype"); @@ -625,7 +627,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { private DocumentUpdate createTensorAssignUpdate() { DocumentUpdate result = new DocumentUpdate(docType, new DocumentId(documentId)); result.addFieldUpdate(FieldUpdate.createAssign(docType.getField(tensorField), - createTensorFieldValue("{{x:0}:2.0}"))); + createTensorFieldValue("{{x:0}:2.0}"))); return result; } diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java index 6f841aac821..6f3abd85521 100644 --- a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java @@ -9,16 +9,17 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.10 + * @author Einar M R Rosenvinge */ public class TemporaryDataTypeTestCase { + @Test public void requireNulls() { - TemporaryDataType type = new TemporaryDataType(0); + TemporaryDataType type = new TemporaryDataType(0, ""); assertThat(type.createFieldValue(new Object()), nullValue()); assertThat(type.createFieldValue(), nullValue()); assertThat(type.getValueClass(), nullValue()); assertThat(type.isValueCompatible(new StringFieldValue("")), is(false)); } + } diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java index 80386141968..c94c917d2ca 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.document.datatypes; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import org.junit.Test; import static org.junit.Assert.assertFalse; @@ -13,8 +14,8 @@ import static org.junit.Assert.assertTrue; */ public class TensorFieldValueTestCase { - private static TensorFieldValue createFieldValue(String tensor) { - return new TensorFieldValue(Tensor.from(tensor)); + private static TensorFieldValue createFieldValue(String tensorString) { + return new TensorFieldValue(Tensor.from(tensorString)); } @Test @@ -23,20 +24,27 @@ public class TensorFieldValueTestCase { } @Test + public void requireThatDifferentTensorTypesWithEmptyValuesAreNotEqual() { + TensorFieldValue field1 = new TensorFieldValue(new TensorType.Builder().mapped("x").build()); + TensorFieldValue field2 = new TensorFieldValue(new TensorType.Builder().indexed("y").build()); + assertFalse(field1.equals(field2)); + } + + @Test public void requireThatDifferentTensorValuesAreNotEqual() { - TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}"); - TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}"); - assertFalse(lhs.equals(rhs)); - assertFalse(lhs.equals(new TensorFieldValue())); + TensorFieldValue field1 = createFieldValue("{{x:0}:2.0}"); + TensorFieldValue field2 = createFieldValue("{{x:0}:3.0}"); + assertFalse(field1.equals(field2)); + assertFalse(field1.equals(new TensorFieldValue(TensorType.empty))); } @Test public void requireThatSameTensorValueIsEqual() { Tensor tensor = Tensor.from("{{x:0}:2.0}"); - TensorFieldValue lhs = new TensorFieldValue(tensor); - TensorFieldValue rhs = new TensorFieldValue(tensor); - assertTrue(lhs.equals(lhs)); - assertTrue(lhs.equals(rhs)); - assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}"))); + TensorFieldValue field1 = new TensorFieldValue(tensor); + TensorFieldValue field2 = new TensorFieldValue(tensor); + assertTrue(field1.equals(field1)); + assertTrue(field1.equals(field2)); + assertTrue(field1.equals(createFieldValue("{{x:0}:2.0}"))); } } diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index ffec7927ab3..a0f993fd2fc 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -10,7 +10,9 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.junit.Test; @@ -27,6 +29,7 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson; */ public class DocumentUpdateJsonSerializerTest { + final static TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build(); final static DocumentTypeManager types = new DocumentTypeManager(); final static JsonFactory parserFactory = new JsonFactory(); final static DocumentType docType = new DocumentType("doctype"); @@ -39,7 +42,7 @@ public class DocumentUpdateJsonSerializerTest { docType.addField(new Field("float_field", DataType.FLOAT)); docType.addField(new Field("double_field", DataType.DOUBLE)); docType.addField(new Field("byte_field", DataType.BYTE)); - docType.addField(new Field("tensor_field", DataType.TENSOR)); + docType.addField(new Field("tensor_field", new TensorDataType(tensorType))); docType.addField(new Field("predicate_field", DataType.PREDICATE)); docType.addField(new Field("raw_field", DataType.RAW)); docType.addField(new Field("int_array", new ArrayDataType(DataType.INT))); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 6c46f743332..34c93580eb3 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -19,6 +19,7 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; @@ -37,7 +38,10 @@ import com.yahoo.document.update.ClearValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.MapValueUpdate; import com.yahoo.document.update.ValueUpdate; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; import org.junit.After; @@ -66,11 +70,12 @@ import static org.junit.Assert.*; /** * Basic test of JSON streams to Vespa document instances. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JsonReaderTestCase { - DocumentTypeManager types; - JsonFactory parserFactory; + + private DocumentTypeManager types; + private JsonFactory parserFactory; @Rule public ExpectedException exception = ExpectedException.none(); @@ -133,7 +138,10 @@ public class JsonReaderTestCase { } { DocumentType x = new DocumentType("testtensor"); - x.addField(new Field("tensorfield", DataType.TENSOR)); + x.addField(new Field("mappedtensorfield", + new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build()))); + x.addField(new Field("indexedtensorfield", + new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build()))); types.registerDocumentType(x); } { @@ -1022,83 +1030,88 @@ public class JsonReaderTestCase { Document doc = createPutWithoutTensor().getDocument(); assertEquals("testtensor", doc.getId().getDocType()); assertEquals("id:unittest:testtensor::0", doc.getId().toString()); - TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield")); + TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("mappedtensorfield")); assertNull(fieldValue); } @Test public void testParsingOfEmptyTensor() { - assertTensorField("{}", createPutWithTensor("{}")); + assertMappedTensorField("tensor(x{},y{}):{}", createPutWithMappedTensor("{}")); } @Test public void testParsingOfTensorWithEmptyDimensions() { - assertTensorField("{}", - createPutWithTensor("{ " - + " \"dimensions\": [] " - + "}")); + assertMappedTensorField("tensor(x{},y{}):{}", + createPutWithMappedTensor("{ " + + " \"dimensions\": [] " + + "}")); } @Test public void testParsingOfTensorWithEmptyCells() { - assertTensorField("{}", - createPutWithTensor("{ " - + " \"cells\": [] " - + "}")); + assertMappedTensorField("tensor(x{},y{}):{}", + createPutWithMappedTensor("{ " + + " \"cells\": [] " + + "}")); } @Test - public void testParsingOfTensorWithCells() { - assertTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " - + " \"value\": 2.0 }, " - + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, " - + " \"value\": 3.0 } " - + " ]" - + "}")); + public void testParsingOfMappedTensorWithCells() { + Tensor tensor = assertMappedTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", + createPutWithMappedTensor("{ " + + " \"cells\": [ " + + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " + + " \"value\": 2.0 }, " + + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, " + + " \"value\": 3.0 } " + + " ]" + + "}")); + assertTrue(tensor instanceof MappedTensor); // any functional instance is fine } @Test - public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() { - assertTensorField("{{x:a,y:b}:2.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"value\": 2.0, " - + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } " - + " ]" - + "}")); + public void testParsingOfIndexedTensorWithCells() { + Tensor tensor = assertTensorField("{{x:0,y:0}:2.0,{x:1,y:0}:3.0}}", + createPutWithTensor("{ " + + " \"cells\": [ " + + " { \"address\": { \"x\": \"0\", \"y\": \"0\" }, " + + " \"value\": 2.0 }, " + + " { \"address\": { \"x\": \"1\", \"y\": \"0\" }, " + + " \"value\": 3.0 } " + + " ]" + + "}", "indexedtensorfield"), "indexedtensorfield"); + assertTrue(tensor instanceof IndexedTensor); // this matters for performance } @Test - public void testParsingOfTensorWithSingleCellWithoutAddress() { - assertTensorField("{{}:2.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"value\": 2.0 } " - + " ]" - + "}")); + public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() { + assertMappedTensorField("{{x:a,y:b}:2.0}", + createPutWithMappedTensor("{ " + + " \"cells\": [ " + + " { \"value\": 2.0, " + + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } " + + " ]" + + "}")); } @Test - public void testParsingOfTensorWithSingleCellWithoutValue() { - assertTensorField("{{x:a}:0.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"address\": { \"x\": \"a\" } } " - + " ]" - + "}")); + public void testAssignUpdateOfEmptyMappedTensor() { + assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithMappedTensor("{}")); } @Test - public void testAssignUpdateOfEmptyTensor() { - assertTensorAssignUpdate("{}", createAssignUpdateWithTensor("{}")); + public void testAssignUpdateOfEmptyIndexedTensor() { + try { + assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithTensor("{}", "indexedtensorfield")); + } + catch (IllegalArgumentException e) { + assertEquals("An indexed tensor must have a value", "Tensor of type tensor(x[],y[]) has no values", e.getMessage()); + } } @Test public void testAssignUpdateOfNullTensor() { - ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithTensor(null)).getValueUpdate(0); + ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithMappedTensor(null)).getValueUpdate(0); assertTrue(clearUpdate != null); assertTrue(clearUpdate.getValue() == null); } @@ -1106,7 +1119,7 @@ public class JsonReaderTestCase { @Test public void testAssignUpdateOfTensorWithCells() { assertTensorAssignUpdate("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", - createAssignUpdateWithTensor("{ " + createAssignUpdateWithMappedTensor("{ " + " \"cells\": [ " + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " + " \"value\": 2.0 }, " @@ -1194,10 +1207,13 @@ public class JsonReaderTestCase { return (DocumentPut) reader.next(); } - private DocumentPut createPutWithTensor(String inputTensor) { + private DocumentPut createPutWithMappedTensor(String inputTensor) { + return createPutWithTensor(inputTensor, "mappedtensorfield"); + } + private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[" - + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": " + + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": " + inputTensor + " }}" + "]")); @@ -1205,20 +1221,27 @@ public class JsonReaderTestCase { return (DocumentPut) reader.next(); } - private DocumentUpdate createAssignUpdateWithTensor(String inputTensor) { + private DocumentUpdate createAssignUpdateWithMappedTensor(String inputTensor) { + return createAssignUpdateWithTensor(inputTensor, "mappedtensorfield"); + } + private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) { InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {" + Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": {" + "\"assign\": " + (inputTensor != null ? inputTensor : "null") + " } } } ]")); JsonReader reader = new JsonReader(types, rawDoc, parserFactory); return (DocumentUpdate) reader.next(); } - private static void assertTensorField(String expectedTensor, DocumentPut put) { + private static Tensor assertMappedTensorField(String expectedTensor, DocumentPut put) { + return assertTensorField(expectedTensor, put, "mappedtensorfield"); + } + private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) { final Document doc = put.getDocument(); assertEquals("testtensor", doc.getId().getDocType()); assertEquals(TENSOR_DOC_ID, doc.getId().toString()); - TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield")); + TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField(tensorFieldName)); assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get()); + return fieldValue.getTensor().get(); } private static void assertTensorAssignUpdate(String expectedTensor, DocumentUpdate update) { @@ -1230,7 +1253,7 @@ public class JsonReaderTestCase { } private static FieldUpdate getTensorField(DocumentUpdate update) { - FieldUpdate fieldUpdate = update.getFieldUpdate("tensorfield"); + FieldUpdate fieldUpdate = update.getFieldUpdate("mappedtensorfield"); assertEquals(1, fieldUpdate.size()); return fieldUpdate; } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index f29c208a113..57b3e088efc 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -19,8 +19,10 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; import org.junit.After; @@ -115,7 +117,8 @@ public class JsonWriterTestCase { } { DocumentType x = new DocumentType("testtensor"); - x.addField(new Field("tensorfield", DataType.TENSOR)); + TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build(); + x.addField(new Field("tensorfield", new TensorDataType(tensorType))); types.registerDocumentType(x); } } @@ -298,7 +301,7 @@ public class JsonWriterTestCase { @Test public void testWritingOfEmptyTensor() throws IOException { - assertTensorRoundTripEquality("{}","{ \"cells\": [{\"address\": {}, \"value\": 0.0}] }"); + assertTensorRoundTripEquality("{}","{ \"cells\": [] }"); } @Test @@ -321,20 +324,12 @@ public class JsonWriterTestCase { } @Test - public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException { - assertTensorRoundTripEquality("{ " - + " \"cells\": [ " - + " { \"address\": {}, \"value\": 2.0 } " - + " ]" - + "}"); - } - - @Test public void testWritingOfTensorFieldValueWithoutTensor() throws IOException { - DocumentType tensorType = types.getDocumentType("testtensor"); + DocumentType documentTypeWithTensor = types.getDocumentType("testtensor"); String docId = "id:unittest:testtensor::0"; - Document doc = new Document(tensorType, docId); - doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue()); + Document doc = new Document(documentTypeWithTensor, docId); + Field tensorField = documentTypeWithTensor.getField("tensorfield"); + doc.setFieldValue(tensorField, new TensorFieldValue(((TensorDataType)tensorField.getDataType()).getTensorType())); assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc)); } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java index f3987085e32..7e3fabc30fb 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java @@ -2,6 +2,7 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.io.GrowableByteBuffer; import java.io.IOException; diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java index 22cb35ae937..ae61bb3cf6f 100644 --- a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java @@ -4,8 +4,10 @@ package com.yahoo.document.serialization; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import org.junit.Test; import java.io.IOException; @@ -19,30 +21,31 @@ import static org.junit.Assert.assertEquals; */ public class TensorFieldValueSerializationTestCase { + private final static TensorType tensorType = new TensorType.Builder().mapped("dimX").mapped("dimY").build(); private final static String TENSOR_FIELD = "my_tensor"; private final static String TENSOR_FILES = "src/test/resources/tensor/"; - private final static TestDocumentFactory docFactory = - new TestDocumentFactory(createDocType(), "id:test:my_type::foo"); + private final static TestDocumentFactory docFactory = new TestDocumentFactory(createDocType(), + "id:test:my_type::foo"); private static DocumentType createDocType() { DocumentType type = new DocumentType("my_type"); - type.addField(TENSOR_FIELD, DataType.TENSOR); + type.addField(TENSOR_FIELD, new TensorDataType(tensorType)); return type; } @Test public void requireThatTensorFieldValueIsSerializedAndDeserialized() { - assertSerialization(new TensorFieldValue()); - assertSerialization(createTensor("{}")); - assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); + assertSerialization(new TensorFieldValue(tensorType)); + assertSerialization(createTensor(tensorType, "{}")); + assertSerialization(createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); } @Test public void requireThatSerializationMatchesCpp() throws IOException { - assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue()); - assertSerializationMatchesCpp("empty_tensor", createTensor("{}")); + assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue(tensorType)); + assertSerializationMatchesCpp("empty_tensor", createTensor(tensorType, "{}")); assertSerializationMatchesCpp("multi_cell_tensor", - createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); + createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); } private static void assertSerialization(TensorFieldValue tensor) { @@ -60,8 +63,8 @@ public class TensorFieldValueSerializationTestCase { SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory); } - private static TensorFieldValue createTensor(String tensor) { - return new TensorFieldValue(Tensor.from(tensor)); + private static TensorFieldValue createTensor(TensorType type, String tensorCellString) { + return new TensorFieldValue(Tensor.from(type, tensorCellString)); } } diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java index 3cfbcac5b62..3dc6ebd1403 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java @@ -12,6 +12,7 @@ import com.yahoo.document.predicate.FeatureRange; import com.yahoo.document.predicate.FeatureSet; import com.yahoo.document.predicate.Predicate; import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.tensor.TensorType; import org.apache.commons.codec.binary.Base64; import org.junit.Test; @@ -75,8 +76,8 @@ public class VespaXmlFieldReaderTestCase { @Test public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception { - assertThrows(new Field("my_tensor", DataType.TENSOR), "", - "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); + assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "", + "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); } private class MockedReaderFixture { diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java index 8730265c80d..8a5fabde9ea 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java @@ -7,7 +7,9 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.tensor.TensorType; import org.junit.Ignore; import org.junit.Test; @@ -215,8 +217,8 @@ public class VespaXmlUpdateReaderTestCase { @Test public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception { - assertThrows(new Field("my_tensor", DataType.TENSOR), "<assign field='my_tensor'></assign>", - "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); + assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "<assign field='my_tensor'></assign>", + "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); } private static void assertThrows(Field field, String fieldXml, String expected) throws Exception { diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp Binary files differindex 365182b14eb..2c15c152558 100644 --- a/document/src/test/resources/tensor/empty_tensor__cpp +++ b/document/src/test/resources/tensor/empty_tensor__cpp diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java Binary files differindex 1fbade8c785..2c15c152558 100644 --- a/document/src/test/resources/tensor/empty_tensor__java +++ b/document/src/test/resources/tensor/empty_tensor__java diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp Binary files differindex c0b2b3a165a..d923fc10559 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__cpp +++ b/document/src/test/resources/tensor/multi_cell_tensor__cpp diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 98aa3f3ae3e..7e5e978f0d1 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -811,7 +811,7 @@ void checkDeserialization(const string &name, std::unique_ptr<Tensor> tensor) { TEST("Require that tensor deserialization matches Java") { checkDeserialization("non_existing_tensor", std::unique_ptr<Tensor>()); - checkDeserialization("empty_tensor", createTensor({{{}, 0.0}}, {})); + checkDeserialization("empty_tensor", createTensor({}, {})); checkDeserialization("multi_cell_tensor", createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 }, {{{"dimX", "ccc"}, diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 267273cfa21..6ababfc19fb 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -60,6 +60,10 @@ datatype[].structtype[].field[].id[].id int ## one specified in config. datatype[].structtype[].field[].datatype int +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +datatype[].structtype[].field[].detailedtype string default="" + ## Specify a document type to inherit datatype[].structtype[].inherits[].name string diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 5e0c5e4e528..21cde02ad54 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -28,7 +28,7 @@ documenttype[].inherits[].id int documenttype[].datatype[].id int ## This is the type of the datatype. -documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE} +documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE, TENSOR} ## This is the id of the datatype of the element in the array. documenttype[].datatype[].array.element.id int default=0 @@ -82,6 +82,10 @@ documenttype[].datatype[].sstruct.field[].id_v6 int ## or one of its inherited document types. documenttype[].datatype[].sstruct.field[].datatype int +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +documenttype[].datatype[].sstruct.field[].detailedtype string default="" + ## The id of the annotation type. documenttype[].annotationtype[].id int |