aboutsummaryrefslogtreecommitdiffstats
path: root/document
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2017-01-16 23:16:32 +0100
committerGitHub <noreply@github.com>2017-01-16 23:16:32 +0100
commit9efbe90ee02b6b56f1a1208614c20d4b053a3135 (patch)
tree86dc5fcbd4e2244c7a82c2019a77635d33f7b87a /document
parent7305314406270fe663b7ef301cd6ad04714dcf9b (diff)
Revert "Revert "Revert "Bratseth/tensor type info in documents"""
Diffstat (limited to 'document')
-rw-r--r--document/src/main/java/com/yahoo/document/ArrayDataType.java3
-rw-r--r--document/src/main/java/com/yahoo/document/DataType.java69
-rw-r--r--document/src/main/java/com/yahoo/document/Document.java5
-rw-r--r--document/src/main/java/com/yahoo/document/DocumentTypeManager.java55
-rw-r--r--document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java22
-rw-r--r--document/src/main/java/com/yahoo/document/Field.java2
-rw-r--r--document/src/main/java/com/yahoo/document/PrimitiveDataType.java3
-rw-r--r--document/src/main/java/com/yahoo/document/TemporaryDataType.java11
-rw-r--r--document/src/main/java/com/yahoo/document/TensorDataType.java50
-rw-r--r--document/src/main/java/com/yahoo/document/datatypes/Array.java2
-rw-r--r--document/src/main/java/com/yahoo/document/datatypes/Struct.java2
-rw-r--r--document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java2
-rw-r--r--document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java57
-rw-r--r--document/src/main/java/com/yahoo/document/json/JsonReader.java46
-rw-r--r--document/src/main/java/com/yahoo/document/serialization/DocumentReader.java2
-rw-r--r--document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java8
-rw-r--r--document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java6
-rw-r--r--document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java7
-rw-r--r--document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java30
-rw-r--r--document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java5
-rw-r--r--document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java137
-rw-r--r--document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java23
-rw-r--r--document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java1
-rw-r--r--document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java25
-rw-r--r--document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java5
-rw-r--r--document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java6
-rw-r--r--document/src/test/resources/tensor/empty_tensor__cppbin64 -> 54 bytes
-rw-r--r--document/src/test/resources/tensor/empty_tensor__javabin64 -> 62 bytes
-rw-r--r--document/src/tests/serialization/vespadocumentserializer_test.cpp2
-rw-r--r--document/src/vespa/document/config/documentmanager.def4
-rw-r--r--document/src/vespa/document/config/documenttypes.def6
31 files changed, 248 insertions, 348 deletions
diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java
index 4b24bb1ae00..640bd94bd1c 100644
--- a/document/src/main/java/com/yahoo/document/ArrayDataType.java
+++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java
@@ -8,10 +8,9 @@ import java.util.ArrayList;
import java.util.List;
/**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class ArrayDataType extends CollectionDataType {
-
// The global class identifier shared with C++.
public static int classId = registerClass(Ids.document + 54, ArrayDataType.class);
diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java
index 184340f295e..51af799efd9 100644
--- a/document/src/main/java/com/yahoo/document/DataType.java
+++ b/document/src/main/java/com/yahoo/document/DataType.java
@@ -14,7 +14,6 @@ import com.yahoo.document.datatypes.Raw;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.TensorFieldValue;
import com.yahoo.document.datatypes.UriFieldValue;
-import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.objects.Identifiable;
import com.yahoo.vespa.objects.Ids;
import com.yahoo.vespa.objects.ObjectVisitor;
@@ -47,30 +46,40 @@ public abstract class DataType extends Identifiable implements Serializable, Com
public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory());
public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory());
public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory());
+ // ARRAY is type 6, but never used, array IDs are generated
+ // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class);
public final static DocumentType DOCUMENT = new DocumentType("document");
+ // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class);
public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory());
+ // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class);
+ // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory());
public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory());
- public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory());
- public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately
- // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor
-
- // Tags are converted to weightedset<string> when reading the search definition TODO: Remove it
+ // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated
+ // Tags are converted to weightedset&lt;string&gt; when reading the search definition
public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true);
+ // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class);
+ // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now..
+ // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType
+ public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory());
+ public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory());
public static int lastPredefinedDataTypeId() {
return 21;
}
- /** Set to true when this type is registered in a type manager. From that time we should refuse changes. */
+ /**
+ * Set to true when this type is registered in a type manager. From that time we should refuse changes.
+ */
private boolean registered = false;
private String name;
- /** The id of this type */
+ /**
+ * The id of this type
+ */
private int dataTypeId;
static final private CopyOnWriteHashMap<Pair, Constructor> constructorCache = new CopyOnWriteHashMap<>();
-
/**
* Creates a datatype
*
@@ -87,21 +96,26 @@ public abstract class DataType extends Identifiable implements Serializable, Com
return (DataType)super.clone();
}
- void setRegistered() {
+ public void setRegistered() {
registered = true;
}
+
public boolean isRegistered() {
return registered;
}
- /** Creates a new, empty FieldValue of this type */
+ /**
+ * Creates a new, empty FieldValue of this type.
+ *
+ * @return a new, empty FieldValue of this type.
+ */
public abstract FieldValue createFieldValue();
/**
- * Creates a field value by reflection
- *
- * @param arg the value of the newly created field value
- * @return a fully constructed value
+ * This will try to create the object by reflection. This can be very expensive
+ * so some might discourage that.
+ * @param arg The constructor argument.
+ * @return Fully constructed value.
*/
protected FieldValue createByReflection(Object arg) {
Class<?> valClass = getValueClass();
@@ -127,8 +141,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com
/**
* Creates a new FieldValue of this type, with the given value.
*
- * @param arg the value of the new FieldValue
- * @return a new FieldValue of this type, with the given value
+ * @param arg the value that the new FieldValue shall have.
+ * @return A new FieldValue of this type, with the given value.
*/
public FieldValue createFieldValue(Object arg) {
if (arg == null) {
@@ -197,17 +211,12 @@ public abstract class DataType extends Identifiable implements Serializable, Com
return new WeightedSetDataType(type, createIfNonExistent, removeIfZero);
}
- /** Returns the given tensor type as a DataType */
- public static TensorDataType getTensor(TensorType type) {
- return new TensorDataType(type);
- }
-
public String getName() {
return name;
}
/**
- * Sets the name of this data type. WARNING! Do not use!
+ * Sets the name of this data type.&nbsp;WARNING! Do not use!
*
* @param name the name of this data type.
*/
@@ -220,7 +229,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com
}
/**
- * Sets the ID of this data type. WARNING! Do not use!
+ * Sets the ID of this data type.&nbsp;WARNING! Do not use!
*
* @param id the ID of this data type.
*/
@@ -229,7 +238,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com
}
/**
- * Registers this type in the given document manager.
+ * Registeres this type in the given document manager.
*
* @param manager the DocumentTypeManager to register in.
*/
@@ -246,12 +255,14 @@ public abstract class DataType extends Identifiable implements Serializable, Com
}
public boolean equals(Object other) {
- if (!(other instanceof DataType)) return false;
+ if (!(other instanceof DataType)) {
+ return false;
+ }
DataType type = (DataType)other;
return (name.equals(type.name) && dataTypeId == type.dataTypeId);
}
- public String toString() {
+ public java.lang.String toString() {
return "datatype " + name + " (code: " + dataTypeId + ")";
}
@@ -267,8 +278,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com
*/
public FieldPath buildFieldPath(String fieldPathString) {
if (fieldPathString.length() > 0) {
- throw new IllegalArgumentException("Datatype " + toString() +
- " does not support further recursive structure: " + fieldPathString);
+ throw new IllegalArgumentException(
+ "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString);
}
return new FieldPath();
}
diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java
index e1d912b4e51..34c952e1cec 100644
--- a/document/src/main/java/com/yahoo/document/Document.java
+++ b/document/src/main/java/com/yahoo/document/Document.java
@@ -26,7 +26,7 @@ import java.util.Map;
* be removed soon.
*
* @author bratseth
- * @author Einar M R Rosenvinge
+ * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class Document extends StructuredFieldValue {
@@ -267,9 +267,7 @@ public class Document extends StructuredFieldValue {
}
/** Returns true if the argument is a document which has the same set of values */
- @Override
public boolean equals(Object o) {
- if (o == this) return true;
if (!(o instanceof Document)) return false;
Document other = (Document) o;
return (super.equals(o) && docId.equals(other.docId) &&
@@ -396,5 +394,4 @@ public class Document extends StructuredFieldValue {
comp = body.compareTo(otherValue.body);
return comp;
}
-
}
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
index 1b3370cfa66..0de7fe60500 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
@@ -12,7 +12,6 @@ import com.yahoo.document.serialization.DocumentDeserializer;
import com.yahoo.document.serialization.DocumentDeserializerFactory;
import com.yahoo.document.serialization.VespaDocumentDeserializer42;
import com.yahoo.io.GrowableByteBuffer;
-import com.yahoo.tensor.TensorType;
import java.lang.reflect.Modifier;
import java.util.*;
@@ -29,23 +28,13 @@ import java.util.logging.Logger;
* datatypes, for instance displaying the data type in human-readable form
* or as XML.
*
- * @author Thomas Gundersen
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
*/
public class DocumentTypeManager {
private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName());
private ConfigSubscriber subscriber;
- // *Configured data types* (not built-in/primitive) indexed by their id
- //
- // *Primitive* data types are always available and have a single id.
- //
- // *Built-in dynamic* types: The tensor type.
- // Any tensor type has the same id and is always available just like primitive types.
- // However, unlike primitive types, each tensor type is a separate DataType instance
- // (which carries additional type information (detailedType) supplied at runtime).
- // The reason for this is that we want the data type to stay the same when we change the detailed tensor type
- // to maintain compatibility on (some) tensor type changes.
private Map<Integer, DataType> dataTypes = new LinkedHashMap<>();
private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>();
private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry();
@@ -96,7 +85,6 @@ public class DocumentTypeManager {
}
public boolean hasDataType(String name) {
- if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present
for (DataType type : dataTypes.values()) {
if (type.getName().equalsIgnoreCase(name)) {
return true;
@@ -106,14 +94,10 @@ public class DocumentTypeManager {
}
public boolean hasDataType(int code) {
- if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present
return dataTypes.containsKey(code);
}
public DataType getDataType(String name) {
- if (name.startsWith("tensor(")) // built-in dynamic
- return new TensorDataType(TensorType.fromSpec(name));
-
List<DataType> foundTypes = new ArrayList<>();
for (DataType type : dataTypes.values()) {
if (type.getName().equalsIgnoreCase(name)) {
@@ -141,19 +125,7 @@ public class DocumentTypeManager {
return foundTypes.get(0);
}
- public DataType getDataType(int code) { return getDataType(code, ""); }
-
- /**
- * Return a data type instance
- *
- * @param code the code of the data type to return, which must be either built in or present in this manager
- * @param detailedType detailed type information, or the empty string if none
- * @return the appropriate DataType instance
- */
- public DataType getDataType(int code, String detailedType) {
- if (code == DataType.tensorDataTypeCode) // built-in dynamic
- return new TensorDataType(TensorType.fromSpec(detailedType));
-
+ public DataType getDataType(int code) {
DataType type = dataTypes.get(code);
if (type == null) {
StringBuilder types=new StringBuilder();
@@ -166,11 +138,11 @@ public class DocumentTypeManager {
}
}
- DataType getDataTypeAndReturnTemporary(int code, String detailedType) {
+ DataType getDataTypeAndReturnTemporary(int code) {
if (hasDataType(code)) {
- return getDataType(code, detailedType);
+ return getDataType(code);
}
- return new TemporaryDataType(code, detailedType);
+ return new TemporaryDataType(code);
}
/**
@@ -184,11 +156,9 @@ public class DocumentTypeManager {
/**
* Register a single datatype. Re-registering an existing, but equal, datatype is ok.
- *
* @param type The datatype to register
*/
void registerSingleType(DataType type) {
- if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly
if (dataTypes.containsKey(type.getId())) {
DataType existingType = dataTypes.get(type.getId());
if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType))
@@ -218,7 +188,7 @@ public class DocumentTypeManager {
dataTypes.remove(existingType.getId());
} else {
throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered "
- + type + ", but already uses id " + type.getId());
+ + type + ", but already uses id " + type.getId());
}
}
@@ -280,8 +250,7 @@ public class DocumentTypeManager {
}
/**
- * Returns a read only view of the registered data types
- *
+ * A read only view of the registered data types
* @return collection of types
*/
public Collection<DataType> getDataTypes() {
@@ -349,7 +318,7 @@ public class DocumentTypeManager {
for (Field field : structDataType.getFieldsThisTypeOnly()) {
DataType fieldType = field.getDataType();
if (fieldType instanceof TemporaryDataType) {
- field.setDataType(getDataType(fieldType.getCode(), ((TemporaryDataType)fieldType).getDetailedType()));
+ field.setDataType(getDataType(fieldType.getCode()));
} else {
if (!seenStructs.contains(fieldType)) {
replaceTemporaryTypes(fieldType, seenStructs);
@@ -360,7 +329,7 @@ public class DocumentTypeManager {
private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List<DataType> seenStructs) {
if (collectionDataType.getNestedType() instanceof TemporaryDataType) {
- collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode(), ""));
+ collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode()));
} else {
replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs);
}
@@ -368,13 +337,13 @@ public class DocumentTypeManager {
private void replaceTemporaryTypesInMap(MapDataType mapDataType, List<DataType> seenStructs) {
if (mapDataType.getValueType() instanceof TemporaryDataType) {
- mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode(), ""));
+ mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode()));
} else {
replaceTemporaryTypes(mapDataType.getValueType(), seenStructs);
}
if (mapDataType.getKeyType() instanceof TemporaryDataType) {
- mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode(), ""));
+ mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode()));
} else {
replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs);
}
@@ -382,7 +351,7 @@ public class DocumentTypeManager {
private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List<DataType> seenStructs) {
if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) {
- weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode(), ""));
+ weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode()));
} else {
replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs);
}
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 7b5f1392032..9d7139e6226 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -13,7 +13,7 @@ import java.util.logging.Logger;
/**
* Configures the Vespa document manager from a config id.
*
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{
@@ -82,21 +82,21 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
try {
for (Object o : thisDataType.arraytype()) {
DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o;
- DataType nestedType = manager.getDataType(array.datatype(), "");
+ DataType nestedType = manager.getDataType(array.datatype());
ArrayDataType type = new ArrayDataType(nestedType, id);
manager.register(type);
}
for (Object o : thisDataType.maptype()) {
DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o;
- DataType keyType = manager.getDataType(map.keytype(), "");
- DataType valType = manager.getDataType(map.valtype(), "");
+ DataType keyType = manager.getDataType(map.keytype());
+ DataType valType = manager.getDataType(map.valtype());
MapDataType type = new MapDataType(keyType, valType, id);
manager.register(type);
}
for (Object o : thisDataType.weightedsettype()) {
DocumentmanagerConfig.Datatype.Weightedsettype wset =
(DocumentmanagerConfig.Datatype.Weightedsettype) o;
- DataType nestedType = manager.getDataType(wset.datatype(), "");
+ DataType nestedType = manager.getDataType(wset.datatype());
WeightedSetDataType type = new WeightedSetDataType(
nestedType, wset.createifnonexistant(), wset.removeifzero(), id);
manager.register(type);
@@ -114,8 +114,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
DocumentmanagerConfig.Datatype.Structtype.Field field =
(DocumentmanagerConfig.Datatype.Structtype.Field) j;
DataType fieldType = (field.datatype() == id)
- ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype())
- : manager.getDataType(field.datatype(), field.detailedtype());
+ ? manager.getDataTypeAndReturnTemporary(field.datatype())
+ : manager.getDataType(field.datatype());
if (field.id().size() == 1) {
type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true));
@@ -127,8 +127,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
}
for (Object o : thisDataType.documenttype()) {
DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o;
- StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), "");
- StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), "");
+ StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct());
+ StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct());
for (Field field : body.getFields()) {
field.setHeader(false);
}
@@ -210,7 +210,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) {
for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) {
AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id());
- DataType payload = manager.getDataType(annType.datatype(), "");
+ DataType payload = manager.getDataType(annType.datatype());
if (!payload.equals(DataType.NONE)) {
annotationType.setDataType(payload);
}
@@ -234,7 +234,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
int id = thisDataType.id();
for (Object o : thisDataType.structtype()) {
DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o;
- StructDataType thisStruct = (StructDataType) manager.getDataType(id, "");
+ StructDataType thisStruct = (StructDataType) manager.getDataType(id);
for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) {
StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name());
diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java
index b80d92e0e4d..86543916b42 100644
--- a/document/src/main/java/com/yahoo/document/Field.java
+++ b/document/src/main/java/com/yahoo/document/Field.java
@@ -13,7 +13,7 @@ import java.io.Serializable;
* A name and type. Fields are contained in document types to describe their fields,
* but is also used to represent name/type pairs which are not part of document types.
*
- * @author Thomas Gundersen
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
* @author bratseth
*/
public class Field extends FieldBase implements FieldSet, Comparable, Serializable {
diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
index 23bf4b43ccf..a0024bb0497 100644
--- a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
+++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
@@ -8,10 +8,9 @@ import com.yahoo.vespa.objects.ObjectVisitor;
import java.util.Objects;
/**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class PrimitiveDataType extends DataType {
-
public static abstract class Factory {
public abstract FieldValue create();
}
diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
index a0c6fcb889d..da65dde72da 100644
--- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java
+++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
@@ -4,15 +4,11 @@ package com.yahoo.document;
import com.yahoo.document.datatypes.FieldValue;
/**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
class TemporaryDataType extends DataType {
-
- private final String detailedType;
-
- TemporaryDataType(int dataTypeId, String detailedType) {
+ TemporaryDataType(int dataTypeId) {
super("temporary_" + dataTypeId, dataTypeId);
- this.detailedType = detailedType;
}
@Override
@@ -29,7 +25,4 @@ class TemporaryDataType extends DataType {
public boolean isValueCompatible(FieldValue value) {
return false;
}
-
- String getDetailedType() { return detailedType; }
-
}
diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java
deleted file mode 100644
index dbaf6ee7763..00000000000
--- a/document/src/main/java/com/yahoo/document/TensorDataType.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.yahoo.document;
-
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.datatypes.TensorFieldValue;
-import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.objects.Ids;
-
-/**
- * A DataType containing a tensor type
- *
- * @author bratseth
- */
-public class TensorDataType extends DataType {
-
- private final TensorType tensorType;
-
- // The global class identifier shared with C++.
- public static int classId = registerClass(Ids.document + 59, TensorDataType.class);
-
- public TensorDataType(TensorType tensorType) {
- super(tensorType.toString(), DataType.tensorDataTypeCode);
- this.tensorType = tensorType;
- }
-
- public TensorDataType clone() {
- return (TensorDataType)super.clone();
- }
-
- @Override
- public FieldValue createFieldValue() {
- return new TensorFieldValue(tensorType);
- }
-
- @Override
- public Class<? extends TensorFieldValue> getValueClass() {
- return TensorFieldValue.class;
- }
-
- @Override
- public boolean isValueCompatible(FieldValue value) {
- if (value == null) return false;
- if ( ! TensorFieldValue.class.isAssignableFrom(value.getClass())) return false;
- TensorFieldValue tensorValue = (TensorFieldValue)value;
- return tensorValue.getDataType().getTensorType().isAssignableTo(tensorType);
- }
-
- /** Returns the type of the tensor this field can hold */
- public TensorType getTensorType() { return tensorType; }
-
-}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java
index 09fb8b71db1..66cc472de69 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Array.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java
@@ -16,7 +16,7 @@ import java.util.*;
/**
* FieldValue which encapsulates a Array value
*
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public final class Array<T extends FieldValue> extends CollectionFieldValue<T> implements List<T> {
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
index b5920cb4758..5a01dc33aa1 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
@@ -215,6 +215,7 @@ public class Struct extends StructuredFieldValue {
if (!super.equals(o)) return false;
Struct struct = (Struct) o;
+
return values.equals(struct.values);
}
@@ -387,5 +388,4 @@ public class Struct extends StructuredFieldValue {
}
return fieldType.cast(fieldValue);
}
-
}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
index 7957d9b812f..b4585a2188d 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
@@ -10,7 +10,7 @@ import java.util.List;
import java.util.Map;
/**
- * @author Håkon Humberset
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
*/
public abstract class StructuredFieldValue extends CompositeFieldValue {
diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
index 9d8e9a83b5e..bee29478219 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
@@ -3,12 +3,11 @@ package com.yahoo.document.datatypes;
import com.yahoo.document.DataType;
import com.yahoo.document.Field;
-import com.yahoo.document.TensorDataType;
+import com.yahoo.document.PrimitiveDataType;
import com.yahoo.document.serialization.FieldReader;
import com.yahoo.document.serialization.FieldWriter;
import com.yahoo.document.serialization.XmlStream;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import java.util.Objects;
import java.util.Optional;
@@ -21,18 +20,12 @@ import java.util.Optional;
public class TensorFieldValue extends FieldValue {
private Optional<Tensor> tensor;
-
- private final TensorDataType dataType;
- /** Create an empty tensor field value */
- public TensorFieldValue(TensorType type) {
- this.dataType = new TensorDataType(type);
- this.tensor = Optional.empty();
+ public TensorFieldValue() {
+ tensor = Optional.empty();
}
- /** Create a tensor field value containing the given tensor */
public TensorFieldValue(Tensor tensor) {
- this.dataType = new TensorDataType(tensor.type());
this.tensor = Optional.of(tensor);
}
@@ -41,8 +34,8 @@ public class TensorFieldValue extends FieldValue {
}
@Override
- public TensorDataType getDataType() {
- return dataType;
+ public DataType getDataType() {
+ return DataType.TENSOR;
}
@Override
@@ -58,23 +51,16 @@ public class TensorFieldValue extends FieldValue {
@Override
public void assign(Object o) {
if (o == null) {
- assignTensor(Optional.empty());
+ tensor = Optional.empty();
} else if (o instanceof Tensor) {
- assignTensor(Optional.of((Tensor)o));
+ tensor = Optional.of((Tensor)o);
} else if (o instanceof TensorFieldValue) {
- assignTensor(((TensorFieldValue)o).getTensor());
+ tensor = ((TensorFieldValue)o).getTensor();
} else {
throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" +
- o.getClass().getName() + "'.");
+ o.getClass().getName() + "'.");
}
}
-
- public void assignTensor(Optional<Tensor> tensor) {
- if (tensor.isPresent() && ! dataType.getTensorType().isAssignableTo(tensor.get().type()))
- throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() +
- " to field of type " + dataType.getTensorType());
- this.tensor = tensor;
- }
@Override
public void serialize(Field field, FieldWriter writer) {
@@ -88,14 +74,27 @@ public class TensorFieldValue extends FieldValue {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if ( ! (o instanceof TensorFieldValue)) return false;
-
- TensorFieldValue other = (TensorFieldValue)o;
- if ( ! dataType.getTensorType().equals(other.dataType.getTensorType())) return false;
- if ( ! tensor.equals(other.tensor)) return false;
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TensorFieldValue)) {
+ return false;
+ }
+ TensorFieldValue rhs = (TensorFieldValue)o;
+ if (!Objects.equals(tensor, rhs.tensor)) {
+ return false;
+ }
return true;
}
+ public static PrimitiveDataType.Factory getFactory() {
+ return new PrimitiveDataType.Factory() {
+
+ @Override
+ public FieldValue create() {
+ return new TensorFieldValue();
+ }
+ };
+ }
}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index e0740338aae..b1c635d7641 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -583,41 +583,71 @@ public class JsonReader {
}
private void fillTensor(TensorFieldValue tensorFieldValue) {
- Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
expectObjectStart(buffer.currentToken());
int initNesting = buffer.nesting();
+ Tensor.Builder tensorBuilder = null;
// read tensor cell fields and ignore everything else
for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
if (TENSOR_CELLS.equals(buffer.currentName()))
- readTensorCells(tensorBuilder);
+ tensorBuilder = readTensorCells(tensorBuilder);
}
expectObjectEnd(buffer.currentToken());
+ if (tensorBuilder == null) // no cells + no type: empty tensor type
+ tensorBuilder = Tensor.Builder.of(TensorType.empty);
tensorFieldValue.assign(tensorBuilder.build());
}
- private void readTensorCells(Tensor.Builder tensorBuilder) {
+ private Tensor.Builder readTensorCells(Tensor.Builder tensorBuilder) {
expectArrayStart(buffer.currentToken());
int initNesting = buffer.nesting();
- for (buffer.next(); buffer.nesting() >= initNesting; buffer.next())
- readTensorCell(tensorBuilder);
+ for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+ tensorBuilder = readTensorCell(tensorBuilder);
+ }
expectCompositeEnd(buffer.currentToken());
+ return tensorBuilder;
}
- private void readTensorCell(Tensor.Builder tensorBuilder) {
+ private Tensor.Builder readTensorCell(Tensor.Builder tensorBuilder) {
expectObjectStart(buffer.currentToken());
int initNesting = buffer.nesting();
double cellValue = 0.0;
- Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell();
+ Tensor.Builder.CellBuilder cellBuilder = null;
for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
String currentName = buffer.currentName();
if (TENSOR_ADDRESS.equals(currentName)) {
- readTensorAddress(cellBuilder);
+ if (tensorBuilder != null) {
+ cellBuilder = tensorBuilder.cell();
+ readTensorAddress(cellBuilder);
+ }
+ else { // gnarly temporary path to create a type on the fly TODO; Remove when we always have a type
+ expectObjectStart(buffer.currentToken());
+ int initNesting2 = buffer.nesting();
+ List<Pair<String,String>> entries = new ArrayList<>();
+ for (buffer.next(); buffer.nesting() >= initNesting2; buffer.next()) {
+ String dimension = buffer.currentName();
+ String label = buffer.currentText();
+ entries.add(new Pair<>(dimension, label));
+ }
+ TensorType.Builder typeBuilder = new TensorType.Builder();
+ for (Pair<String,String> entry : entries)
+ typeBuilder.mapped(entry.getFirst());
+ tensorBuilder = Tensor.Builder.of(typeBuilder.build());
+ cellBuilder = tensorBuilder.cell();
+ for (Pair<String,String> entry : entries)
+ cellBuilder.label(entry.getFirst(), entry.getSecond());
+ expectObjectEnd(buffer.currentToken());
+ }
} else if (TENSOR_VALUE.equals(currentName)) {
cellValue = Double.valueOf(buffer.currentText());
}
}
expectObjectEnd(buffer.currentToken());
+ if (tensorBuilder == null) { // no content TODO; This will go away with the above
+ tensorBuilder = Tensor.Builder.of(TensorType.empty);
+ cellBuilder = tensorBuilder.cell();
+ }
cellBuilder.value(cellValue);
+ return tensorBuilder;
}
private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) {
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
index 5f1b227790b..52a62caf296 100644
--- a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
@@ -9,7 +9,7 @@ import com.yahoo.document.DocumentTypeManager;
/**
* This interface is used to implement custom deserialization of document updates.
*
- * @author Ravi Sharma
+ * @author <a href="mailto:ravishar@yahoo-inc.com">Ravi Sharma</a>
* @author baldersheim
*/
public interface DocumentReader {
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
index 6e9495b1437..753008de7e0 100644
--- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
@@ -115,8 +115,8 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
// Verify that we have correct version
version = getShort(null);
if (version < 6 || version > Document.SERIALIZED_VERSION) {
- throw new DeserializationException("Unknown version " + version + ", expected " +
- Document.SERIALIZED_VERSION + ".");
+ throw new DeserializationException(
+ "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + ".");
}
int dataLength = 0;
@@ -278,7 +278,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
int encodedTensorLength = buf.getInt1_4Bytes();
if (encodedTensorLength > 0) {
byte[] encodedTensor = getBytes(null, encodedTensorLength);
- value.assign(TypedBinaryFormat.decode(value.getDataType().getTensorType(), encodedTensor));
+ value.assign(TypedBinaryFormat.decode(null, encodedTensor)); // TODO: Pass type
} else {
value.clear();
}
@@ -328,7 +328,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple
fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null)));
}
- // save a reference to the big buffer we're reading from:
+ //save a reference to the big buffer we're reading from:
GrowableByteBuffer bigBuf = buf;
if (version < 7) {
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
index 02773c7dad0..413d1581e58 100644
--- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -10,7 +10,6 @@ import com.yahoo.document.update.FieldUpdate;
import com.yahoo.document.update.ValueUpdate;
import com.yahoo.io.GrowableByteBuffer;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -44,7 +43,6 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
private final String documentId = "doc:something:foooo";
private final String tensorField = "tensorfield";
- private final TensorType tensorType = new TensorType.Builder().mapped("x").build();
private Document createDocument() {
return new Document(docMan.getDocumentType("foobar"), new DocumentId(documentId));
@@ -62,7 +60,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
DataType stringwset = DataType.getWeightedSet(DataType.STRING);
docType.addField(new Field("strwset", stringwset));
- docType.addField(new Field(tensorField, new TensorDataType(tensorType)));
+ docType.addField(new Field(tensorField, DataType.TENSOR));
docMan.register(docType);
docType2 = new DocumentType("otherdoctype");
@@ -627,7 +625,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase {
private DocumentUpdate createTensorAssignUpdate() {
DocumentUpdate result = new DocumentUpdate(docType, new DocumentId(documentId));
result.addFieldUpdate(FieldUpdate.createAssign(docType.getField(tensorField),
- createTensorFieldValue("{{x:0}:2.0}")));
+ createTensorFieldValue("{{x:0}:2.0}")));
return result;
}
diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
index 6f3abd85521..6f841aac821 100644
--- a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
+++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
@@ -9,17 +9,16 @@ import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
/**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
*/
public class TemporaryDataTypeTestCase {
-
@Test
public void requireNulls() {
- TemporaryDataType type = new TemporaryDataType(0, "");
+ TemporaryDataType type = new TemporaryDataType(0);
assertThat(type.createFieldValue(new Object()), nullValue());
assertThat(type.createFieldValue(), nullValue());
assertThat(type.getValueClass(), nullValue());
assertThat(type.isValueCompatible(new StringFieldValue("")), is(false));
}
-
}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
index c94c917d2ca..80386141968 100644
--- a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
+++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
@@ -2,7 +2,6 @@
package com.yahoo.document.datatypes;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import org.junit.Test;
import static org.junit.Assert.assertFalse;
@@ -14,8 +13,8 @@ import static org.junit.Assert.assertTrue;
*/
public class TensorFieldValueTestCase {
- private static TensorFieldValue createFieldValue(String tensorString) {
- return new TensorFieldValue(Tensor.from(tensorString));
+ private static TensorFieldValue createFieldValue(String tensor) {
+ return new TensorFieldValue(Tensor.from(tensor));
}
@Test
@@ -24,27 +23,20 @@ public class TensorFieldValueTestCase {
}
@Test
- public void requireThatDifferentTensorTypesWithEmptyValuesAreNotEqual() {
- TensorFieldValue field1 = new TensorFieldValue(new TensorType.Builder().mapped("x").build());
- TensorFieldValue field2 = new TensorFieldValue(new TensorType.Builder().indexed("y").build());
- assertFalse(field1.equals(field2));
- }
-
- @Test
public void requireThatDifferentTensorValuesAreNotEqual() {
- TensorFieldValue field1 = createFieldValue("{{x:0}:2.0}");
- TensorFieldValue field2 = createFieldValue("{{x:0}:3.0}");
- assertFalse(field1.equals(field2));
- assertFalse(field1.equals(new TensorFieldValue(TensorType.empty)));
+ TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}");
+ TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}");
+ assertFalse(lhs.equals(rhs));
+ assertFalse(lhs.equals(new TensorFieldValue()));
}
@Test
public void requireThatSameTensorValueIsEqual() {
Tensor tensor = Tensor.from("{{x:0}:2.0}");
- TensorFieldValue field1 = new TensorFieldValue(tensor);
- TensorFieldValue field2 = new TensorFieldValue(tensor);
- assertTrue(field1.equals(field1));
- assertTrue(field1.equals(field2));
- assertTrue(field1.equals(createFieldValue("{{x:0}:2.0}")));
+ TensorFieldValue lhs = new TensorFieldValue(tensor);
+ TensorFieldValue rhs = new TensorFieldValue(tensor);
+ assertTrue(lhs.equals(lhs));
+ assertTrue(lhs.equals(rhs));
+ assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}")));
}
}
diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
index a0f993fd2fc..ffec7927ab3 100644
--- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
+++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
@@ -10,9 +10,7 @@ import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.Field;
import com.yahoo.document.MapDataType;
import com.yahoo.document.PositionDataType;
-import com.yahoo.document.TensorDataType;
import com.yahoo.document.WeightedSetDataType;
-import com.yahoo.tensor.TensorType;
import com.yahoo.text.Utf8;
import org.junit.Test;
@@ -29,7 +27,6 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson;
*/
public class DocumentUpdateJsonSerializerTest {
- final static TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build();
final static DocumentTypeManager types = new DocumentTypeManager();
final static JsonFactory parserFactory = new JsonFactory();
final static DocumentType docType = new DocumentType("doctype");
@@ -42,7 +39,7 @@ public class DocumentUpdateJsonSerializerTest {
docType.addField(new Field("float_field", DataType.FLOAT));
docType.addField(new Field("double_field", DataType.DOUBLE));
docType.addField(new Field("byte_field", DataType.BYTE));
- docType.addField(new Field("tensor_field", new TensorDataType(tensorType)));
+ docType.addField(new Field("tensor_field", DataType.TENSOR));
docType.addField(new Field("predicate_field", DataType.PREDICATE));
docType.addField(new Field("raw_field", DataType.RAW));
docType.addField(new Field("int_array", new ArrayDataType(DataType.INT)));
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 34c93580eb3..6c46f743332 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -19,7 +19,6 @@ import com.yahoo.document.Field;
import com.yahoo.document.MapDataType;
import com.yahoo.document.PositionDataType;
import com.yahoo.document.StructDataType;
-import com.yahoo.document.TensorDataType;
import com.yahoo.document.WeightedSetDataType;
import com.yahoo.document.datatypes.Array;
import com.yahoo.document.datatypes.FieldValue;
@@ -38,10 +37,7 @@ import com.yahoo.document.update.ClearValueUpdate;
import com.yahoo.document.update.FieldUpdate;
import com.yahoo.document.update.MapValueUpdate;
import com.yahoo.document.update.ValueUpdate;
-import com.yahoo.tensor.IndexedTensor;
-import com.yahoo.tensor.MappedTensor;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import com.yahoo.text.Utf8;
import org.apache.commons.codec.binary.Base64;
import org.junit.After;
@@ -70,12 +66,11 @@ import static org.junit.Assert.*;
/**
* Basic test of JSON streams to Vespa document instances.
*
- * @author Steinar Knutsen
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
*/
public class JsonReaderTestCase {
-
- private DocumentTypeManager types;
- private JsonFactory parserFactory;
+ DocumentTypeManager types;
+ JsonFactory parserFactory;
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -138,10 +133,7 @@ public class JsonReaderTestCase {
}
{
DocumentType x = new DocumentType("testtensor");
- x.addField(new Field("mappedtensorfield",
- new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build())));
- x.addField(new Field("indexedtensorfield",
- new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build())));
+ x.addField(new Field("tensorfield", DataType.TENSOR));
types.registerDocumentType(x);
}
{
@@ -1030,88 +1022,83 @@ public class JsonReaderTestCase {
Document doc = createPutWithoutTensor().getDocument();
assertEquals("testtensor", doc.getId().getDocType());
assertEquals("id:unittest:testtensor::0", doc.getId().toString());
- TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("mappedtensorfield"));
+ TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
assertNull(fieldValue);
}
@Test
public void testParsingOfEmptyTensor() {
- assertMappedTensorField("tensor(x{},y{}):{}", createPutWithMappedTensor("{}"));
+ assertTensorField("{}", createPutWithTensor("{}"));
}
@Test
public void testParsingOfTensorWithEmptyDimensions() {
- assertMappedTensorField("tensor(x{},y{}):{}",
- createPutWithMappedTensor("{ "
- + " \"dimensions\": [] "
- + "}"));
+ assertTensorField("{}",
+ createPutWithTensor("{ "
+ + " \"dimensions\": [] "
+ + "}"));
}
@Test
public void testParsingOfTensorWithEmptyCells() {
- assertMappedTensorField("tensor(x{},y{}):{}",
- createPutWithMappedTensor("{ "
- + " \"cells\": [] "
- + "}"));
+ assertTensorField("{}",
+ createPutWithTensor("{ "
+ + " \"cells\": [] "
+ + "}"));
}
@Test
- public void testParsingOfMappedTensorWithCells() {
- Tensor tensor = assertMappedTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
- createPutWithMappedTensor("{ "
- + " \"cells\": [ "
- + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
- + " \"value\": 2.0 }, "
- + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, "
- + " \"value\": 3.0 } "
- + " ]"
- + "}"));
- assertTrue(tensor instanceof MappedTensor); // any functional instance is fine
+ public void testParsingOfTensorWithCells() {
+ assertTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
+ createPutWithTensor("{ "
+ + " \"cells\": [ "
+ + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+ + " \"value\": 2.0 }, "
+ + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, "
+ + " \"value\": 3.0 } "
+ + " ]"
+ + "}"));
}
@Test
- public void testParsingOfIndexedTensorWithCells() {
- Tensor tensor = assertTensorField("{{x:0,y:0}:2.0,{x:1,y:0}:3.0}}",
- createPutWithTensor("{ "
- + " \"cells\": [ "
- + " { \"address\": { \"x\": \"0\", \"y\": \"0\" }, "
- + " \"value\": 2.0 }, "
- + " { \"address\": { \"x\": \"1\", \"y\": \"0\" }, "
- + " \"value\": 3.0 } "
- + " ]"
- + "}", "indexedtensorfield"), "indexedtensorfield");
- assertTrue(tensor instanceof IndexedTensor); // this matters for performance
+ public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
+ assertTensorField("{{x:a,y:b}:2.0}",
+ createPutWithTensor("{ "
+ + " \"cells\": [ "
+ + " { \"value\": 2.0, "
+ + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
+ + " ]"
+ + "}"));
}
@Test
- public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
- assertMappedTensorField("{{x:a,y:b}:2.0}",
- createPutWithMappedTensor("{ "
- + " \"cells\": [ "
- + " { \"value\": 2.0, "
- + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
- + " ]"
- + "}"));
+ public void testParsingOfTensorWithSingleCellWithoutAddress() {
+ assertTensorField("{{}:2.0}",
+ createPutWithTensor("{ "
+ + " \"cells\": [ "
+ + " { \"value\": 2.0 } "
+ + " ]"
+ + "}"));
}
@Test
- public void testAssignUpdateOfEmptyMappedTensor() {
- assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithMappedTensor("{}"));
+ public void testParsingOfTensorWithSingleCellWithoutValue() {
+ assertTensorField("{{x:a}:0.0}",
+ createPutWithTensor("{ "
+ + " \"cells\": [ "
+ + " { \"address\": { \"x\": \"a\" } } "
+ + " ]"
+ + "}"));
}
@Test
- public void testAssignUpdateOfEmptyIndexedTensor() {
- try {
- assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithTensor("{}", "indexedtensorfield"));
- }
- catch (IllegalArgumentException e) {
- assertEquals("An indexed tensor must have a value", "Tensor of type tensor(x[],y[]) has no values", e.getMessage());
- }
+ public void testAssignUpdateOfEmptyTensor() {
+ assertTensorAssignUpdate("{}", createAssignUpdateWithTensor("{}"));
}
@Test
public void testAssignUpdateOfNullTensor() {
- ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithMappedTensor(null)).getValueUpdate(0);
+ ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithTensor(null)).getValueUpdate(0);
assertTrue(clearUpdate != null);
assertTrue(clearUpdate.getValue() == null);
}
@@ -1119,7 +1106,7 @@ public class JsonReaderTestCase {
@Test
public void testAssignUpdateOfTensorWithCells() {
assertTensorAssignUpdate("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}",
- createAssignUpdateWithMappedTensor("{ "
+ createAssignUpdateWithTensor("{ "
+ " \"cells\": [ "
+ " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+ " \"value\": 2.0 }, "
@@ -1207,13 +1194,10 @@ public class JsonReaderTestCase {
return (DocumentPut) reader.next();
}
- private DocumentPut createPutWithMappedTensor(String inputTensor) {
- return createPutWithTensor(inputTensor, "mappedtensorfield");
- }
- private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) {
+ private DocumentPut createPutWithTensor(String inputTensor) {
InputStream rawDoc = new ByteArrayInputStream(
Utf8.toBytes("["
- + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": "
+ + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": "
+ inputTensor
+ " }}"
+ "]"));
@@ -1221,27 +1205,20 @@ public class JsonReaderTestCase {
return (DocumentPut) reader.next();
}
- private DocumentUpdate createAssignUpdateWithMappedTensor(String inputTensor) {
- return createAssignUpdateWithTensor(inputTensor, "mappedtensorfield");
- }
- private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) {
+ private DocumentUpdate createAssignUpdateWithTensor(String inputTensor) {
InputStream rawDoc = new ByteArrayInputStream(
- Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": {"
+ Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {"
+ "\"assign\": " + (inputTensor != null ? inputTensor : "null") + " } } } ]"));
JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
return (DocumentUpdate) reader.next();
}
- private static Tensor assertMappedTensorField(String expectedTensor, DocumentPut put) {
- return assertTensorField(expectedTensor, put, "mappedtensorfield");
- }
- private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) {
+ private static void assertTensorField(String expectedTensor, DocumentPut put) {
final Document doc = put.getDocument();
assertEquals("testtensor", doc.getId().getDocType());
assertEquals(TENSOR_DOC_ID, doc.getId().toString());
- TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField(tensorFieldName));
+ TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get());
- return fieldValue.getTensor().get();
}
private static void assertTensorAssignUpdate(String expectedTensor, DocumentUpdate update) {
@@ -1253,7 +1230,7 @@ public class JsonReaderTestCase {
}
private static FieldUpdate getTensorField(DocumentUpdate update) {
- FieldUpdate fieldUpdate = update.getFieldUpdate("mappedtensorfield");
+ FieldUpdate fieldUpdate = update.getFieldUpdate("tensorfield");
assertEquals(1, fieldUpdate.size());
return fieldUpdate;
}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index 57b3e088efc..f29c208a113 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -19,10 +19,8 @@ import com.yahoo.document.Field;
import com.yahoo.document.MapDataType;
import com.yahoo.document.PositionDataType;
import com.yahoo.document.StructDataType;
-import com.yahoo.document.TensorDataType;
import com.yahoo.document.WeightedSetDataType;
import com.yahoo.document.datatypes.TensorFieldValue;
-import com.yahoo.tensor.TensorType;
import com.yahoo.text.Utf8;
import org.apache.commons.codec.binary.Base64;
import org.junit.After;
@@ -117,8 +115,7 @@ public class JsonWriterTestCase {
}
{
DocumentType x = new DocumentType("testtensor");
- TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build();
- x.addField(new Field("tensorfield", new TensorDataType(tensorType)));
+ x.addField(new Field("tensorfield", DataType.TENSOR));
types.registerDocumentType(x);
}
}
@@ -301,7 +298,7 @@ public class JsonWriterTestCase {
@Test
public void testWritingOfEmptyTensor() throws IOException {
- assertTensorRoundTripEquality("{}","{ \"cells\": [] }");
+ assertTensorRoundTripEquality("{}","{ \"cells\": [{\"address\": {}, \"value\": 0.0}] }");
}
@Test
@@ -324,12 +321,20 @@ public class JsonWriterTestCase {
}
@Test
+ public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException {
+ assertTensorRoundTripEquality("{ "
+ + " \"cells\": [ "
+ + " { \"address\": {}, \"value\": 2.0 } "
+ + " ]"
+ + "}");
+ }
+
+ @Test
public void testWritingOfTensorFieldValueWithoutTensor() throws IOException {
- DocumentType documentTypeWithTensor = types.getDocumentType("testtensor");
+ DocumentType tensorType = types.getDocumentType("testtensor");
String docId = "id:unittest:testtensor::0";
- Document doc = new Document(documentTypeWithTensor, docId);
- Field tensorField = documentTypeWithTensor.getField("tensorfield");
- doc.setFieldValue(tensorField, new TensorFieldValue(((TensorDataType)tensorField.getDataType()).getTensorType()));
+ Document doc = new Document(tensorType, docId);
+ doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue());
assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc));
}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
index 7e3fabc30fb..f3987085e32 100644
--- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
@@ -2,7 +2,6 @@
package com.yahoo.document.serialization;
import com.yahoo.document.Document;
-import com.yahoo.document.datatypes.TensorFieldValue;
import com.yahoo.io.GrowableByteBuffer;
import java.io.IOException;
diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
index ae61bb3cf6f..22cb35ae937 100644
--- a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
+++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
@@ -4,10 +4,8 @@ package com.yahoo.document.serialization;
import com.yahoo.document.DataType;
import com.yahoo.document.Document;
import com.yahoo.document.DocumentType;
-import com.yahoo.document.TensorDataType;
import com.yahoo.document.datatypes.TensorFieldValue;
import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
import org.junit.Test;
import java.io.IOException;
@@ -21,31 +19,30 @@ import static org.junit.Assert.assertEquals;
*/
public class TensorFieldValueSerializationTestCase {
- private final static TensorType tensorType = new TensorType.Builder().mapped("dimX").mapped("dimY").build();
private final static String TENSOR_FIELD = "my_tensor";
private final static String TENSOR_FILES = "src/test/resources/tensor/";
- private final static TestDocumentFactory docFactory = new TestDocumentFactory(createDocType(),
- "id:test:my_type::foo");
+ private final static TestDocumentFactory docFactory =
+ new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
private static DocumentType createDocType() {
DocumentType type = new DocumentType("my_type");
- type.addField(TENSOR_FIELD, new TensorDataType(tensorType));
+ type.addField(TENSOR_FIELD, DataType.TENSOR);
return type;
}
@Test
public void requireThatTensorFieldValueIsSerializedAndDeserialized() {
- assertSerialization(new TensorFieldValue(tensorType));
- assertSerialization(createTensor(tensorType, "{}"));
- assertSerialization(createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
+ assertSerialization(new TensorFieldValue());
+ assertSerialization(createTensor("{}"));
+ assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
}
@Test
public void requireThatSerializationMatchesCpp() throws IOException {
- assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue(tensorType));
- assertSerializationMatchesCpp("empty_tensor", createTensor(tensorType, "{}"));
+ assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue());
+ assertSerializationMatchesCpp("empty_tensor", createTensor("{}"));
assertSerializationMatchesCpp("multi_cell_tensor",
- createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
+ createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}"));
}
private static void assertSerialization(TensorFieldValue tensor) {
@@ -63,8 +60,8 @@ public class TensorFieldValueSerializationTestCase {
SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory);
}
- private static TensorFieldValue createTensor(TensorType type, String tensorCellString) {
- return new TensorFieldValue(Tensor.from(type, tensorCellString));
+ private static TensorFieldValue createTensor(String tensor) {
+ return new TensorFieldValue(Tensor.from(tensor));
}
}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
index 3dc6ebd1403..3cfbcac5b62 100644
--- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
@@ -12,7 +12,6 @@ import com.yahoo.document.predicate.FeatureRange;
import com.yahoo.document.predicate.FeatureSet;
import com.yahoo.document.predicate.Predicate;
import com.yahoo.document.serialization.DeserializationException;
-import com.yahoo.tensor.TensorType;
import org.apache.commons.codec.binary.Base64;
import org.junit.Test;
@@ -76,8 +75,8 @@ public class VespaXmlFieldReaderTestCase {
@Test
public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception {
- assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "",
- "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+ assertThrows(new Field("my_tensor", DataType.TENSOR), "",
+ "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
}
private class MockedReaderFixture {
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
index 8a5fabde9ea..8730265c80d 100644
--- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
@@ -7,9 +7,7 @@ import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.Field;
import com.yahoo.document.StructDataType;
-import com.yahoo.document.TensorDataType;
import com.yahoo.document.serialization.DeserializationException;
-import com.yahoo.tensor.TensorType;
import org.junit.Ignore;
import org.junit.Test;
@@ -217,8 +215,8 @@ public class VespaXmlUpdateReaderTestCase {
@Test
public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception {
- assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "<assign field='my_tensor'></assign>",
- "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+ assertThrows(new Field("my_tensor", DataType.TENSOR), "<assign field='my_tensor'></assign>",
+ "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
}
private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp
index 2c15c152558..365182b14eb 100644
--- a/document/src/test/resources/tensor/empty_tensor__cpp
+++ b/document/src/test/resources/tensor/empty_tensor__cpp
Binary files differ
diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java
index 2c15c152558..1fbade8c785 100644
--- a/document/src/test/resources/tensor/empty_tensor__java
+++ b/document/src/test/resources/tensor/empty_tensor__java
Binary files differ
diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp
index 27f725bf38f..98aa3f3ae3e 100644
--- a/document/src/tests/serialization/vespadocumentserializer_test.cpp
+++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp
@@ -811,7 +811,7 @@ void checkDeserialization(const string &name, std::unique_ptr<Tensor> tensor) {
TEST("Require that tensor deserialization matches Java") {
checkDeserialization("non_existing_tensor", std::unique_ptr<Tensor>());
- checkDeserialization("empty_tensor", createTensor({}, {"dimX", "dimY"}));
+ checkDeserialization("empty_tensor", createTensor({{{}, 0.0}}, {}));
checkDeserialization("multi_cell_tensor",
createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 },
{{{"dimX", "ccc"},
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
index 6ababfc19fb..267273cfa21 100644
--- a/document/src/vespa/document/config/documentmanager.def
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -60,10 +60,6 @@ datatype[].structtype[].field[].id[].id int
## one specified in config.
datatype[].structtype[].field[].datatype int
-## Additional, optional type information which can be changed without
-## (necessarily) causing field incompatibility
-datatype[].structtype[].field[].detailedtype string default=""
-
## Specify a document type to inherit
datatype[].structtype[].inherits[].name string
diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def
index 21cde02ad54..5e0c5e4e528 100644
--- a/document/src/vespa/document/config/documenttypes.def
+++ b/document/src/vespa/document/config/documenttypes.def
@@ -28,7 +28,7 @@ documenttype[].inherits[].id int
documenttype[].datatype[].id int
## This is the type of the datatype.
-documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE, TENSOR}
+documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE}
## This is the id of the datatype of the element in the array.
documenttype[].datatype[].array.element.id int default=0
@@ -82,10 +82,6 @@ documenttype[].datatype[].sstruct.field[].id_v6 int
## or one of its inherited document types.
documenttype[].datatype[].sstruct.field[].datatype int
-## Additional, optional type information which can be changed without
-## (necessarily) causing field incompatibility
-documenttype[].datatype[].sstruct.field[].detailedtype string default=""
-
## The id of the annotation type.
documenttype[].annotationtype[].id int