From 9efbe90ee02b6b56f1a1208614c20d4b053a3135 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 16 Jan 2017 23:16:32 +0100 Subject: Revert "Revert "Revert "Bratseth/tensor type info in documents""" --- .../java/com/yahoo/document/ArrayDataType.java | 3 +- .../src/main/java/com/yahoo/document/DataType.java | 69 +++++++++++++--------- .../src/main/java/com/yahoo/document/Document.java | 5 +- .../com/yahoo/document/DocumentTypeManager.java | 55 ++++------------- .../document/DocumentTypeManagerConfigurer.java | 22 +++---- .../src/main/java/com/yahoo/document/Field.java | 2 +- .../java/com/yahoo/document/PrimitiveDataType.java | 3 +- .../java/com/yahoo/document/TemporaryDataType.java | 11 +--- .../java/com/yahoo/document/TensorDataType.java | 50 ---------------- .../java/com/yahoo/document/datatypes/Array.java | 2 +- .../java/com/yahoo/document/datatypes/Struct.java | 2 +- .../document/datatypes/StructuredFieldValue.java | 2 +- .../yahoo/document/datatypes/TensorFieldValue.java | 57 +++++++++--------- .../java/com/yahoo/document/json/JsonReader.java | 46 ++++++++++++--- .../document/serialization/DocumentReader.java | 2 +- .../serialization/VespaDocumentDeserializer42.java | 8 +-- 16 files changed, 143 insertions(+), 196 deletions(-) delete mode 100644 document/src/main/java/com/yahoo/document/TensorDataType.java (limited to 'document/src/main/java/com/yahoo') diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java index 4b24bb1ae00..640bd94bd1c 100644 --- a/document/src/main/java/com/yahoo/document/ArrayDataType.java +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -8,10 +8,9 @@ import java.util.ArrayList; import java.util.List; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class ArrayDataType extends CollectionDataType { - // The global class identifier shared with C++. public static int classId = registerClass(Ids.document + 54, ArrayDataType.class); diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 184340f295e..51af799efd9 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -14,7 +14,6 @@ import com.yahoo.document.datatypes.Raw; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.UriFieldValue; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Ids; import com.yahoo.vespa.objects.ObjectVisitor; @@ -47,30 +46,40 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory()); public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory()); public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory()); + // ARRAY is type 6, but never used, array IDs are generated + // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class); public final static DocumentType DOCUMENT = new DocumentType("document"); + // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); + // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class); + // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); - public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); - public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately - // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor - - // Tags are converted to weightedset when reading the search definition TODO: Remove it + // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated + // Tags are converted to weightedset<string> when reading the search definition public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); + // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class); + // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now.. + // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType + public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); + public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory()); public static int lastPredefinedDataTypeId() { return 21; } - /** Set to true when this type is registered in a type manager. From that time we should refuse changes. */ + /** + * Set to true when this type is registered in a type manager. From that time we should refuse changes. + */ private boolean registered = false; private String name; - /** The id of this type */ + /** + * The id of this type + */ private int dataTypeId; static final private CopyOnWriteHashMap constructorCache = new CopyOnWriteHashMap<>(); - /** * Creates a datatype * @@ -87,21 +96,26 @@ public abstract class DataType extends Identifiable implements Serializable, Com return (DataType)super.clone(); } - void setRegistered() { + public void setRegistered() { registered = true; } + public boolean isRegistered() { return registered; } - /** Creates a new, empty FieldValue of this type */ + /** + * Creates a new, empty FieldValue of this type. + * + * @return a new, empty FieldValue of this type. + */ public abstract FieldValue createFieldValue(); /** - * Creates a field value by reflection - * - * @param arg the value of the newly created field value - * @return a fully constructed value + * This will try to create the object by reflection. This can be very expensive + * so some might discourage that. + * @param arg The constructor argument. + * @return Fully constructed value. */ protected FieldValue createByReflection(Object arg) { Class valClass = getValueClass(); @@ -127,8 +141,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com /** * Creates a new FieldValue of this type, with the given value. * - * @param arg the value of the new FieldValue - * @return a new FieldValue of this type, with the given value + * @param arg the value that the new FieldValue shall have. + * @return A new FieldValue of this type, with the given value. */ public FieldValue createFieldValue(Object arg) { if (arg == null) { @@ -197,17 +211,12 @@ public abstract class DataType extends Identifiable implements Serializable, Com return new WeightedSetDataType(type, createIfNonExistent, removeIfZero); } - /** Returns the given tensor type as a DataType */ - public static TensorDataType getTensor(TensorType type) { - return new TensorDataType(type); - } - public String getName() { return name; } /** - * Sets the name of this data type. WARNING! Do not use! + * Sets the name of this data type. WARNING! Do not use! * * @param name the name of this data type. */ @@ -220,7 +229,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Sets the ID of this data type. WARNING! Do not use! + * Sets the ID of this data type. WARNING! Do not use! * * @param id the ID of this data type. */ @@ -229,7 +238,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Registers this type in the given document manager. + * Registeres this type in the given document manager. * * @param manager the DocumentTypeManager to register in. */ @@ -246,12 +255,14 @@ public abstract class DataType extends Identifiable implements Serializable, Com } public boolean equals(Object other) { - if (!(other instanceof DataType)) return false; + if (!(other instanceof DataType)) { + return false; + } DataType type = (DataType)other; return (name.equals(type.name) && dataTypeId == type.dataTypeId); } - public String toString() { + public java.lang.String toString() { return "datatype " + name + " (code: " + dataTypeId + ")"; } @@ -267,8 +278,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com */ public FieldPath buildFieldPath(String fieldPathString) { if (fieldPathString.length() > 0) { - throw new IllegalArgumentException("Datatype " + toString() + - " does not support further recursive structure: " + fieldPathString); + throw new IllegalArgumentException( + "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString); } return new FieldPath(); } diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index e1d912b4e51..34c952e1cec 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -26,7 +26,7 @@ import java.util.Map; * be removed soon. * * @author bratseth - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class Document extends StructuredFieldValue { @@ -267,9 +267,7 @@ public class Document extends StructuredFieldValue { } /** Returns true if the argument is a document which has the same set of values */ - @Override public boolean equals(Object o) { - if (o == this) return true; if (!(o instanceof Document)) return false; Document other = (Document) o; return (super.equals(o) && docId.equals(other.docId) && @@ -396,5 +394,4 @@ public class Document extends StructuredFieldValue { comp = body.compareTo(otherValue.body); return comp; } - } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index 1b3370cfa66..0de7fe60500 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -12,7 +12,6 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.VespaDocumentDeserializer42; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.TensorType; import java.lang.reflect.Modifier; import java.util.*; @@ -29,23 +28,13 @@ import java.util.logging.Logger; * datatypes, for instance displaying the data type in human-readable form * or as XML. * - * @author Thomas Gundersen + * @author Thomas Gundersen */ public class DocumentTypeManager { private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); private ConfigSubscriber subscriber; - // *Configured data types* (not built-in/primitive) indexed by their id - // - // *Primitive* data types are always available and have a single id. - // - // *Built-in dynamic* types: The tensor type. - // Any tensor type has the same id and is always available just like primitive types. - // However, unlike primitive types, each tensor type is a separate DataType instance - // (which carries additional type information (detailedType) supplied at runtime). - // The reason for this is that we want the data type to stay the same when we change the detailed tensor type - // to maintain compatibility on (some) tensor type changes. private Map dataTypes = new LinkedHashMap<>(); private Map documentTypes = new LinkedHashMap<>(); private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry(); @@ -96,7 +85,6 @@ public class DocumentTypeManager { } public boolean hasDataType(String name) { - if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { return true; @@ -106,14 +94,10 @@ public class DocumentTypeManager { } public boolean hasDataType(int code) { - if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present return dataTypes.containsKey(code); } public DataType getDataType(String name) { - if (name.startsWith("tensor(")) // built-in dynamic - return new TensorDataType(TensorType.fromSpec(name)); - List foundTypes = new ArrayList<>(); for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { @@ -141,19 +125,7 @@ public class DocumentTypeManager { return foundTypes.get(0); } - public DataType getDataType(int code) { return getDataType(code, ""); } - - /** - * Return a data type instance - * - * @param code the code of the data type to return, which must be either built in or present in this manager - * @param detailedType detailed type information, or the empty string if none - * @return the appropriate DataType instance - */ - public DataType getDataType(int code, String detailedType) { - if (code == DataType.tensorDataTypeCode) // built-in dynamic - return new TensorDataType(TensorType.fromSpec(detailedType)); - + public DataType getDataType(int code) { DataType type = dataTypes.get(code); if (type == null) { StringBuilder types=new StringBuilder(); @@ -166,11 +138,11 @@ public class DocumentTypeManager { } } - DataType getDataTypeAndReturnTemporary(int code, String detailedType) { + DataType getDataTypeAndReturnTemporary(int code) { if (hasDataType(code)) { - return getDataType(code, detailedType); + return getDataType(code); } - return new TemporaryDataType(code, detailedType); + return new TemporaryDataType(code); } /** @@ -184,11 +156,9 @@ public class DocumentTypeManager { /** * Register a single datatype. Re-registering an existing, but equal, datatype is ok. - * * @param type The datatype to register */ void registerSingleType(DataType type) { - if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly if (dataTypes.containsKey(type.getId())) { DataType existingType = dataTypes.get(type.getId()); if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType)) @@ -218,7 +188,7 @@ public class DocumentTypeManager { dataTypes.remove(existingType.getId()); } else { throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered " - + type + ", but already uses id " + type.getId()); + + type + ", but already uses id " + type.getId()); } } @@ -280,8 +250,7 @@ public class DocumentTypeManager { } /** - * Returns a read only view of the registered data types - * + * A read only view of the registered data types * @return collection of types */ public Collection getDataTypes() { @@ -349,7 +318,7 @@ public class DocumentTypeManager { for (Field field : structDataType.getFieldsThisTypeOnly()) { DataType fieldType = field.getDataType(); if (fieldType instanceof TemporaryDataType) { - field.setDataType(getDataType(fieldType.getCode(), ((TemporaryDataType)fieldType).getDetailedType())); + field.setDataType(getDataType(fieldType.getCode())); } else { if (!seenStructs.contains(fieldType)) { replaceTemporaryTypes(fieldType, seenStructs); @@ -360,7 +329,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List seenStructs) { if (collectionDataType.getNestedType() instanceof TemporaryDataType) { - collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode(), "")); + collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode())); } else { replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs); } @@ -368,13 +337,13 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInMap(MapDataType mapDataType, List seenStructs) { if (mapDataType.getValueType() instanceof TemporaryDataType) { - mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode(), "")); + mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode())); } else { replaceTemporaryTypes(mapDataType.getValueType(), seenStructs); } if (mapDataType.getKeyType() instanceof TemporaryDataType) { - mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode(), "")); + mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode())); } else { replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs); } @@ -382,7 +351,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List seenStructs) { if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) { - weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode(), "")); + weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode())); } else { replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs); } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 7b5f1392032..9d7139e6226 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; /** * Configures the Vespa document manager from a config id. * - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber{ @@ -82,21 +82,21 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub try { for (Object o : thisDataType.arraytype()) { DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o; - DataType nestedType = manager.getDataType(array.datatype(), ""); + DataType nestedType = manager.getDataType(array.datatype()); ArrayDataType type = new ArrayDataType(nestedType, id); manager.register(type); } for (Object o : thisDataType.maptype()) { DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o; - DataType keyType = manager.getDataType(map.keytype(), ""); - DataType valType = manager.getDataType(map.valtype(), ""); + DataType keyType = manager.getDataType(map.keytype()); + DataType valType = manager.getDataType(map.valtype()); MapDataType type = new MapDataType(keyType, valType, id); manager.register(type); } for (Object o : thisDataType.weightedsettype()) { DocumentmanagerConfig.Datatype.Weightedsettype wset = (DocumentmanagerConfig.Datatype.Weightedsettype) o; - DataType nestedType = manager.getDataType(wset.datatype(), ""); + DataType nestedType = manager.getDataType(wset.datatype()); WeightedSetDataType type = new WeightedSetDataType( nestedType, wset.createifnonexistant(), wset.removeifzero(), id); manager.register(type); @@ -114,8 +114,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentmanagerConfig.Datatype.Structtype.Field field = (DocumentmanagerConfig.Datatype.Structtype.Field) j; DataType fieldType = (field.datatype() == id) - ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype()) - : manager.getDataType(field.datatype(), field.detailedtype()); + ? manager.getDataTypeAndReturnTemporary(field.datatype()) + : manager.getDataType(field.datatype()); if (field.id().size() == 1) { type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); @@ -127,8 +127,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } for (Object o : thisDataType.documenttype()) { DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o; - StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); - StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), ""); + StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct()); + StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct()); for (Field field : body.getFields()) { field.setHeader(false); } @@ -210,7 +210,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) { for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id()); - DataType payload = manager.getDataType(annType.datatype(), ""); + DataType payload = manager.getDataType(annType.datatype()); if (!payload.equals(DataType.NONE)) { annotationType.setDataType(payload); } @@ -234,7 +234,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub int id = thisDataType.id(); for (Object o : thisDataType.structtype()) { DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; - StructDataType thisStruct = (StructDataType) manager.getDataType(id, ""); + StructDataType thisStruct = (StructDataType) manager.getDataType(id); for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index b80d92e0e4d..86543916b42 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -13,7 +13,7 @@ import java.io.Serializable; * A name and type. Fields are contained in document types to describe their fields, * but is also used to represent name/type pairs which are not part of document types. * - * @author Thomas Gundersen + * @author Thomas Gundersen * @author bratseth */ public class Field extends FieldBase implements FieldSet, Comparable, Serializable { diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java index 23bf4b43ccf..a0024bb0497 100644 --- a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java +++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java @@ -8,10 +8,9 @@ import com.yahoo.vespa.objects.ObjectVisitor; import java.util.Objects; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class PrimitiveDataType extends DataType { - public static abstract class Factory { public abstract FieldValue create(); } diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java index a0c6fcb889d..da65dde72da 100644 --- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java +++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java @@ -4,15 +4,11 @@ package com.yahoo.document; import com.yahoo.document.datatypes.FieldValue; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ class TemporaryDataType extends DataType { - - private final String detailedType; - - TemporaryDataType(int dataTypeId, String detailedType) { + TemporaryDataType(int dataTypeId) { super("temporary_" + dataTypeId, dataTypeId); - this.detailedType = detailedType; } @Override @@ -29,7 +25,4 @@ class TemporaryDataType extends DataType { public boolean isValueCompatible(FieldValue value) { return false; } - - String getDetailedType() { return detailedType; } - } diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java deleted file mode 100644 index dbaf6ee7763..00000000000 --- a/document/src/main/java/com/yahoo/document/TensorDataType.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.yahoo.document; - -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.objects.Ids; - -/** - * A DataType containing a tensor type - * - * @author bratseth - */ -public class TensorDataType extends DataType { - - private final TensorType tensorType; - - // The global class identifier shared with C++. - public static int classId = registerClass(Ids.document + 59, TensorDataType.class); - - public TensorDataType(TensorType tensorType) { - super(tensorType.toString(), DataType.tensorDataTypeCode); - this.tensorType = tensorType; - } - - public TensorDataType clone() { - return (TensorDataType)super.clone(); - } - - @Override - public FieldValue createFieldValue() { - return new TensorFieldValue(tensorType); - } - - @Override - public Class getValueClass() { - return TensorFieldValue.class; - } - - @Override - public boolean isValueCompatible(FieldValue value) { - if (value == null) return false; - if ( ! TensorFieldValue.class.isAssignableFrom(value.getClass())) return false; - TensorFieldValue tensorValue = (TensorFieldValue)value; - return tensorValue.getDataType().getTensorType().isAssignableTo(tensorType); - } - - /** Returns the type of the tensor this field can hold */ - public TensorType getTensorType() { return tensorType; } - -} diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 09fb8b71db1..66cc472de69 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -16,7 +16,7 @@ import java.util.*; /** * FieldValue which encapsulates a Array value * - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public final class Array extends CollectionFieldValue implements List { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index b5920cb4758..5a01dc33aa1 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -215,6 +215,7 @@ public class Struct extends StructuredFieldValue { if (!super.equals(o)) return false; Struct struct = (Struct) o; + return values.equals(struct.values); } @@ -387,5 +388,4 @@ public class Struct extends StructuredFieldValue { } return fieldType.cast(fieldValue); } - } diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java index 7957d9b812f..b4585a2188d 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -10,7 +10,7 @@ import java.util.List; import java.util.Map; /** - * @author Håkon Humberset + * @author Håkon Humberset */ public abstract class StructuredFieldValue extends CompositeFieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index 9d8e9a83b5e..bee29478219 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -3,12 +3,11 @@ package com.yahoo.document.datatypes; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.document.TensorDataType; +import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.serialization.FieldReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.document.serialization.XmlStream; import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import java.util.Objects; import java.util.Optional; @@ -21,18 +20,12 @@ import java.util.Optional; public class TensorFieldValue extends FieldValue { private Optional tensor; - - private final TensorDataType dataType; - /** Create an empty tensor field value */ - public TensorFieldValue(TensorType type) { - this.dataType = new TensorDataType(type); - this.tensor = Optional.empty(); + public TensorFieldValue() { + tensor = Optional.empty(); } - /** Create a tensor field value containing the given tensor */ public TensorFieldValue(Tensor tensor) { - this.dataType = new TensorDataType(tensor.type()); this.tensor = Optional.of(tensor); } @@ -41,8 +34,8 @@ public class TensorFieldValue extends FieldValue { } @Override - public TensorDataType getDataType() { - return dataType; + public DataType getDataType() { + return DataType.TENSOR; } @Override @@ -58,23 +51,16 @@ public class TensorFieldValue extends FieldValue { @Override public void assign(Object o) { if (o == null) { - assignTensor(Optional.empty()); + tensor = Optional.empty(); } else if (o instanceof Tensor) { - assignTensor(Optional.of((Tensor)o)); + tensor = Optional.of((Tensor)o); } else if (o instanceof TensorFieldValue) { - assignTensor(((TensorFieldValue)o).getTensor()); + tensor = ((TensorFieldValue)o).getTensor(); } else { throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" + - o.getClass().getName() + "'."); + o.getClass().getName() + "'."); } } - - public void assignTensor(Optional tensor) { - if (tensor.isPresent() && ! dataType.getTensorType().isAssignableTo(tensor.get().type())) - throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() + - " to field of type " + dataType.getTensorType()); - this.tensor = tensor; - } @Override public void serialize(Field field, FieldWriter writer) { @@ -88,14 +74,27 @@ public class TensorFieldValue extends FieldValue { @Override public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof TensorFieldValue)) return false; - - TensorFieldValue other = (TensorFieldValue)o; - if ( ! dataType.getTensorType().equals(other.dataType.getTensorType())) return false; - if ( ! tensor.equals(other.tensor)) return false; + if (this == o) { + return true; + } + if (!(o instanceof TensorFieldValue)) { + return false; + } + TensorFieldValue rhs = (TensorFieldValue)o; + if (!Objects.equals(tensor, rhs.tensor)) { + return false; + } return true; } + public static PrimitiveDataType.Factory getFactory() { + return new PrimitiveDataType.Factory() { + + @Override + public FieldValue create() { + return new TensorFieldValue(); + } + }; + } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index e0740338aae..b1c635d7641 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -583,41 +583,71 @@ public class JsonReader { } private void fillTensor(TensorFieldValue tensorFieldValue) { - Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); + Tensor.Builder tensorBuilder = null; // read tensor cell fields and ignore everything else for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { if (TENSOR_CELLS.equals(buffer.currentName())) - readTensorCells(tensorBuilder); + tensorBuilder = readTensorCells(tensorBuilder); } expectObjectEnd(buffer.currentToken()); + if (tensorBuilder == null) // no cells + no type: empty tensor type + tensorBuilder = Tensor.Builder.of(TensorType.empty); tensorFieldValue.assign(tensorBuilder.build()); } - private void readTensorCells(Tensor.Builder tensorBuilder) { + private Tensor.Builder readTensorCells(Tensor.Builder tensorBuilder) { expectArrayStart(buffer.currentToken()); int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) - readTensorCell(tensorBuilder); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + tensorBuilder = readTensorCell(tensorBuilder); + } expectCompositeEnd(buffer.currentToken()); + return tensorBuilder; } - private void readTensorCell(Tensor.Builder tensorBuilder) { + private Tensor.Builder readTensorCell(Tensor.Builder tensorBuilder) { expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); + Tensor.Builder.CellBuilder cellBuilder = null; for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { String currentName = buffer.currentName(); if (TENSOR_ADDRESS.equals(currentName)) { - readTensorAddress(cellBuilder); + if (tensorBuilder != null) { + cellBuilder = tensorBuilder.cell(); + readTensorAddress(cellBuilder); + } + else { // gnarly temporary path to create a type on the fly TODO; Remove when we always have a type + expectObjectStart(buffer.currentToken()); + int initNesting2 = buffer.nesting(); + List> entries = new ArrayList<>(); + for (buffer.next(); buffer.nesting() >= initNesting2; buffer.next()) { + String dimension = buffer.currentName(); + String label = buffer.currentText(); + entries.add(new Pair<>(dimension, label)); + } + TensorType.Builder typeBuilder = new TensorType.Builder(); + for (Pair entry : entries) + typeBuilder.mapped(entry.getFirst()); + tensorBuilder = Tensor.Builder.of(typeBuilder.build()); + cellBuilder = tensorBuilder.cell(); + for (Pair entry : entries) + cellBuilder.label(entry.getFirst(), entry.getSecond()); + expectObjectEnd(buffer.currentToken()); + } } else if (TENSOR_VALUE.equals(currentName)) { cellValue = Double.valueOf(buffer.currentText()); } } expectObjectEnd(buffer.currentToken()); + if (tensorBuilder == null) { // no content TODO; This will go away with the above + tensorBuilder = Tensor.Builder.of(TensorType.empty); + cellBuilder = tensorBuilder.cell(); + } cellBuilder.value(cellValue); + return tensorBuilder; } private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) { diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java index 5f1b227790b..52a62caf296 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java @@ -9,7 +9,7 @@ import com.yahoo.document.DocumentTypeManager; /** * This interface is used to implement custom deserialization of document updates. * - * @author Ravi Sharma + * @author Ravi Sharma * @author baldersheim */ public interface DocumentReader { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 6e9495b1437..753008de7e0 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -115,8 +115,8 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple // Verify that we have correct version version = getShort(null); if (version < 6 || version > Document.SERIALIZED_VERSION) { - throw new DeserializationException("Unknown version " + version + ", expected " + - Document.SERIALIZED_VERSION + "."); + throw new DeserializationException( + "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + "."); } int dataLength = 0; @@ -278,7 +278,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); - value.assign(TypedBinaryFormat.decode(value.getDataType().getTensorType(), encodedTensor)); + value.assign(TypedBinaryFormat.decode(null, encodedTensor)); // TODO: Pass type } else { value.clear(); } @@ -328,7 +328,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); } - // save a reference to the big buffer we're reading from: + //save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; if (version < 7) { -- cgit v1.2.3