From 585a5b38c5811da950336c26326636773ca289e3 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 12 Jan 2017 22:17:49 +0100 Subject: Revert "Revert "Bratseth/tensor type info in documents"" --- .../yahoo/documentmodel/DataTypeCollection.java | 8 +- .../java/com/yahoo/documentmodel/DataTypeRepo.java | 8 +- .../com/yahoo/documentmodel/DocumentTypeRepo.java | 2 + .../com/yahoo/documentmodel/NewDocumentType.java | 5 - .../com/yahoo/documentmodel/VespaDocumentType.java | 1 - .../searchdefinition/SDDocumentTypeOrderer.java | 5 +- .../yahoo/searchdefinition/document/Attribute.java | 59 ++++----- .../fieldoperation/FieldOperation.java | 1 + .../processing/IndexingValidation.java | 9 +- .../processing/TensorFieldProcessor.java | 15 ++- .../processing/ValidateFieldTypes.java | 20 +-- .../configmodel/producers/DocumentManager.java | 109 ++++++++-------- .../vespa/configmodel/producers/DocumentTypes.java | 90 ++++++++------ .../validation/SearchDataTypeValidator.java | 38 +++--- config-model/src/main/javacc/SDParser.jj | 3 +- .../search_alltypes/searchdefinitions/simple.sd | 2 +- .../src/test/configmodel/types/documentmanager.cfg | 44 ++++++- .../src/test/configmodel/types/documenttypes.cfg | 42 ++++++- .../types/documenttypes_with_doc_field.cfg | 7 +- .../src/test/derived/advanced/documentmanager.cfg | 22 ++++ .../src/test/derived/advanced/index-info.cfg | 28 ++--- .../annotationsimplicitstruct/documentmanager.cfg | 5 + .../annotationsinheritance/documentmanager.cfg | 10 ++ .../annotationsinheritance2/documentmanager.cfg | 6 + .../annotationspolymorphy/documentmanager.cfg | 5 + .../annotationsreference/documentmanager.cfg | 8 ++ .../derived/annotationssimple/documentmanager.cfg | 4 + .../derived/annotationsstruct/documentmanager.cfg | 6 + .../annotationsstructarray/documentmanager.cfg | 6 + .../src/test/derived/arrays/documentmanager.cfg | 9 ++ .../src/test/derived/arrays/index-info.cfg | 126 +++++++++---------- .../derived/attributeprefetch/documentmanager.cfg | 22 ++++ .../src/test/derived/complex/documentmanager.cfg | 29 +++++ .../src/test/derived/complex/rank-profiles.cfg | 2 +- .../derived/documentderiver/documentmanager.cfg | 75 +++++++++++ .../test/derived/emptydefault/documentmanager.cfg | 6 + .../src/test/derived/id/documentmanager.cfg | 5 + .../test/derived/indexswitches/documentmanager.cfg | 8 ++ .../src/test/derived/indexswitches/index-info.cfg | 60 ++++----- .../inheritfromgrandparent/documentmanager.cfg | 6 + .../derived/inheritfromparent/documentmanager.cfg | 8 ++ .../derived/inheritfromparent/documenttypes.cfg | 6 + .../test/derived/mail/onlydoc/documentmanager.cfg | 18 +++ .../prefixexactattribute/documentmanager.cfg | 9 ++ .../src/test/derived/ranktypes/documentmanager.cfg | 9 ++ .../derived/streamingstruct/documentmanager.cfg | 31 +++++ .../streamingstruct/onlydoc/documentmanager.cfg | 26 ++++ .../derived/structanyorder/documentmanager.cfg | 13 ++ .../src/test/derived/tensor/attributes.cfg | 4 +- .../src/test/derived/tensor/documenttypes.cfg | 6 + .../src/test/derived/tensor/rank-profiles.cfg | 18 ++- config-model/src/test/derived/tensor/tensor.sd | 8 +- config-model/src/test/derived/types/attributes.cfg | 2 +- .../src/test/derived/types/documentmanager.cfg | 41 ++++++ config-model/src/test/derived/types/ilscripts.cfg | 2 +- config-model/src/test/derived/types/index-info.cfg | 2 +- config-model/src/test/derived/types/summary.cfg | 2 +- config-model/src/test/derived/types/summarymap.cfg | 2 +- .../src/test/examples/fieldoftypedocument.cfg | 10 ++ config-model/src/test/examples/structresult.cfg | 10 ++ .../FieldOfTypeDocumentTestCase.java | 5 +- .../searchdefinition/NameFieldCheckTestCase.java | 12 +- .../searchdefinition/RankProfileTestCase.java | 8 +- .../com/yahoo/searchdefinition/StructTestCase.java | 14 +-- .../processing/TensorFieldTestCase.java | 8 +- .../SearchDataTypeValidatorTestCase.java | 3 +- .../search/AttributeChangeValidatorTest.java | 25 +++- .../com/yahoo/fs4/test/RankFeaturesTestCase.java | 27 ++-- .../java/com/yahoo/document/ArrayDataType.java | 3 +- .../src/main/java/com/yahoo/document/DataType.java | 69 +++++------ .../src/main/java/com/yahoo/document/Document.java | 5 +- .../com/yahoo/document/DocumentTypeManager.java | 55 +++++++-- .../document/DocumentTypeManagerConfigurer.java | 22 ++-- .../src/main/java/com/yahoo/document/Field.java | 2 +- .../java/com/yahoo/document/PrimitiveDataType.java | 3 +- .../java/com/yahoo/document/TemporaryDataType.java | 11 +- .../java/com/yahoo/document/TensorDataType.java | 50 ++++++++ .../java/com/yahoo/document/datatypes/Array.java | 2 +- .../java/com/yahoo/document/datatypes/Struct.java | 2 +- .../document/datatypes/StructuredFieldValue.java | 2 +- .../yahoo/document/datatypes/TensorFieldValue.java | 57 ++++----- .../java/com/yahoo/document/json/JsonReader.java | 46 ++----- .../document/serialization/DocumentReader.java | 2 +- .../serialization/VespaDocumentDeserializer42.java | 8 +- .../com/yahoo/document/DocumentUpdateTestCase.java | 6 +- .../yahoo/document/TemporaryDataTypeTestCase.java | 7 +- .../datatypes/TensorFieldValueTestCase.java | 30 +++-- .../json/DocumentUpdateJsonSerializerTest.java | 5 +- .../yahoo/document/json/JsonReaderTestCase.java | 137 ++++++++++++--------- .../yahoo/document/json/JsonWriterTestCase.java | 23 ++-- .../serialization/SerializationTestUtils.java | 1 + .../TensorFieldValueSerializationTestCase.java | 25 ++-- .../VespaXmlFieldReaderTestCase.java | 5 +- .../VespaXmlUpdateReaderTestCase.java | 6 +- .../src/test/resources/tensor/empty_tensor__cpp | Bin 54 -> 64 bytes .../src/test/resources/tensor/empty_tensor__java | Bin 62 -> 64 bytes .../test/resources/tensor/multi_cell_tensor__cpp | Bin 107 -> 107 bytes .../serialization/vespadocumentserializer_test.cpp | 2 +- .../src/vespa/document/config/documentmanager.def | 4 + .../src/vespa/document/config/documenttypes.def | 6 +- .../expressions/ArithmeticExpression.java | 8 +- .../expressions/Base64DecodeExpression.java | 4 +- .../expressions/Base64EncodeExpression.java | 4 +- .../expressions/CatExpression.java | 8 +- .../expressions/ClearStateExpression.java | 4 +- .../expressions/EchoExpression.java | 2 +- .../expressions/ExactExpression.java | 2 +- .../indexinglanguage/expressions/Expression.java | 39 +++--- .../expressions/FlattenExpression.java | 4 +- .../expressions/ForEachExpression.java | 14 +-- .../expressions/GetFieldExpression.java | 6 +- .../expressions/GetVarExpression.java | 6 +- .../expressions/GuardExpression.java | 4 +- .../expressions/HexDecodeExpression.java | 4 +- .../expressions/HexEncodeExpression.java | 4 +- .../expressions/HostNameExpression.java | 4 +- .../expressions/IfThenExpression.java | 14 +-- .../expressions/InputExpression.java | 6 +- .../expressions/JoinExpression.java | 6 +- .../expressions/LowerCaseExpression.java | 4 +- .../expressions/NGramExpression.java | 2 +- .../expressions/NowExpression.java | 4 +- .../expressions/OptimizePredicateExpression.java | 10 +- .../expressions/OutputExpression.java | 4 +- .../expressions/ParenthesisExpression.java | 4 +- .../expressions/RandomExpression.java | 4 +- .../expressions/ScriptExpression.java | 8 +- .../expressions/SelectInputExpression.java | 10 +- .../expressions/SetLanguageExpression.java | 2 +- .../expressions/SetValueExpression.java | 4 +- .../expressions/SetVarExpression.java | 8 +- .../expressions/SplitExpression.java | 4 +- .../expressions/StatementExpression.java | 4 +- .../expressions/SubstringExpression.java | 4 +- .../expressions/SwitchExpression.java | 10 +- .../expressions/ThisExpression.java | 2 +- .../expressions/ToArrayExpression.java | 4 +- .../expressions/ToByteExpression.java | 4 +- .../expressions/ToDoubleExpression.java | 4 +- .../expressions/ToFloatExpression.java | 4 +- .../expressions/ToIntegerExpression.java | 4 +- .../expressions/ToLongExpression.java | 4 +- .../expressions/ToPositionExpression.java | 4 +- .../expressions/ToStringExpression.java | 4 +- .../expressions/ToWsetExpression.java | 4 +- .../expressions/TokenizeExpression.java | 2 +- .../expressions/TrimExpression.java | 4 +- .../expressions/ZCurveExpression.java | 4 +- .../ExpressionConverterTestCase.java | 2 +- .../expressions/ForEachTestCase.java | 2 +- .../expressions/SimpleExpression.java | 4 +- .../src/main/java/com/yahoo/tensor/TensorType.java | 19 +++ .../tensor/serialization/SparseBinaryFormat.java | 14 +-- .../tensor/serialization/TypedBinaryFormat.java | 2 +- .../java/com/yahoo/vespa/objects/Identifiable.java | 4 +- .../serialization/DenseBinaryFormatTestCase.java | 1 - 156 files changed, 1419 insertions(+), 747 deletions(-) create mode 100644 document/src/main/java/com/yahoo/document/TensorDataType.java diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java index 9c507d7be05..ff1a582d7fa 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java @@ -9,7 +9,9 @@ import java.util.Collection; * @author baldersheim */ public interface DataTypeCollection { - public DataType getDataType(String name); - public DataType getDataType(int id); - public Collection getTypes(); + + DataType getDataType(String name); + DataType getDataType(int id); + Collection getTypes(); + } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java index 73f8d7f91c5..2c527201ce4 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java @@ -11,6 +11,7 @@ import java.util.Map; * @author baldersheim */ public class DataTypeRepo implements DataTypeCollection { + Map typeById = new LinkedHashMap<>(); Map typeByName = new LinkedHashMap<>(); @@ -25,10 +26,9 @@ public class DataTypeRepo implements DataTypeCollection { public Collection getTypes() { return typeById.values(); } public DataTypeRepo add(DataType type) { - if (typeByName.containsKey(type.getName()) || - typeById.containsKey(type.getId())) - { - throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered."); + if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) { + throw new IllegalStateException("Data type '" + type.getName() + "', id '" + + type.getId() + "' is already registered."); } typeByName.put(type.getName(), type); typeById.put(type.getId(), type); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java index b71a400666a..1abbf8b3ae1 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java @@ -9,6 +9,7 @@ import java.util.Map; * @author baldersheim */ public class DocumentTypeRepo implements DocumentTypeCollection { + final Map typeById = new LinkedHashMap<>(); final Map typeByName = new LinkedHashMap<>(); @@ -36,4 +37,5 @@ public class DocumentTypeRepo implements DocumentTypeCollection { typeById.put(type.getFullName().getId(), type); return this; } + } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index ccf9bdec302..7c594a4e836 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -13,15 +13,10 @@ import com.yahoo.searchdefinition.processing.BuiltInFieldSets; import java.util.*; /** - * TODO: What is this and why? - * * @author baldersheim */ public final class NewDocumentType extends StructuredDataType implements DataTypeCollection { - /** - * TODO: What is this and why? - */ public static final class Name { // TODO: privatize diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java index 793a5fcff6c..09aa55f776b 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java @@ -32,7 +32,6 @@ public class VespaDocumentType { vespa.add(PositionDataType.INSTANCE); vespa.add(DataType.URI); vespa.add(DataType.PREDICATE); - vespa.add(DataType.TENSOR); return vespa; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index 7d8a87cee8c..2a723596255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -11,9 +11,10 @@ import java.util.*; import java.util.logging.Level; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class SDDocumentTypeOrderer { + private Map createdSDTypes = new LinkedHashMap<>(); private Set seenTypes = new LinkedHashSet<>(); List processingOrder = new LinkedList<>(); @@ -119,6 +120,8 @@ public class SDDocumentTypeOrderer { //do nothing } else if (type instanceof PrimitiveDataType) { //do nothing + } else if (type instanceof TensorDataType) { + //do nothing } else { deployLogger.log(Level.WARNING, "Unknown type : " + type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 950ec791368..bc52aa27f4c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -25,10 +25,6 @@ public final class Attribute implements Cloneable, Serializable { private Type type; private CollectionType collectionType; - /** True if only the enum information should be read from this attribute - * (i.e. the actual values are not relevant, only which documents have the - * same values) Used for collapsing and unique. - */ private boolean removeIfZero = false; private boolean createIfNonExistent = false; private boolean enableBitVectors = false; @@ -41,6 +37,8 @@ public final class Attribute implements Cloneable, Serializable { private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD; + + /** This is set if the type of this is TENSOR */ private Optional tensorType = Optional.empty(); private boolean isPosition = false; @@ -104,16 +102,21 @@ public final class Attribute implements Cloneable, Serializable { } /** Creates an attribute with default settings */ - public Attribute(String name,DataType fieldType) { - this(name,convertDataType(fieldType), convertCollectionType(fieldType)); + public Attribute(String name, DataType fieldType) { + this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType)); setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false); setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false); } - public Attribute(String name,Type type, CollectionType collectionType) { + public Attribute(String name, Type type, CollectionType collectionType) { + this(name, type, collectionType, Optional.empty()); + } + + public Attribute(String name, Type type, CollectionType collectionType, Optional tensorType) { this.name=name; setType(type); setCollectionType(collectionType); + this.tensorType = tensorType; } /** @@ -211,44 +214,42 @@ public final class Attribute implements Cloneable, Serializable { } /** Converts to the right attribute type from a field datatype */ - public static CollectionType convertCollectionType(DataType fieldType) { + private static CollectionType convertCollectionType(DataType fieldType) { if (fieldType instanceof ArrayDataType) { return CollectionType.ARRAY; } else if (fieldType instanceof WeightedSetDataType) { return CollectionType.WEIGHTEDSET; + } else if (fieldType instanceof TensorDataType) { + return CollectionType.SINGLE; } else if (fieldType instanceof PrimitiveDataType) { return CollectionType.SINGLE; } else { throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType"); } } + + private static Optional convertTensorType(DataType fieldType) { + if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty(); + return Optional.of(((TensorDataType)fieldType).getTensorType()); + } /** Converts to the right field type from an attribute type */ - public static DataType convertAttrType(Type attrType) { - if (attrType== Type.STRING) { - return DataType.STRING; - } else if (attrType== Type.INTEGER) { - return DataType.INT; - } else if (attrType== Type.LONG) { - return DataType.LONG; - } else if (attrType== Type.FLOAT) { - return DataType.FLOAT; - } else if (attrType== Type.DOUBLE) { - return DataType.DOUBLE; - } else if (attrType == Type.BYTE) { - return DataType.BYTE; - } else if (attrType == Type.PREDICATE) { - return DataType.PREDICATE; - } else if (attrType == Type.TENSOR) { - return DataType.TENSOR; - } else { - throw new IllegalArgumentException("Don't know which attribute type to " + - "convert " + attrType + " to"); + private DataType toDataType(Type attributeType) { + switch (attributeType) { + case STRING : return DataType.STRING; + case INTEGER: return DataType.INT; + case LONG: return DataType.LONG; + case FLOAT: return DataType.FLOAT; + case DOUBLE: return DataType.DOUBLE; + case BYTE: return DataType.BYTE; + case PREDICATE: return DataType.PREDICATE; + case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); + default: throw new IllegalArgumentException("Unknown attribute type " + attributeType); } } public DataType getDataType() { - DataType dataType = Attribute.convertAttrType(type); + DataType dataType = toDataType(type); if (collectionType.equals(Attribute.CollectionType.ARRAY)) { return DataType.getArray(dataType); } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java index 3b28f380b2d..78f0d9a2997 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java @@ -11,6 +11,7 @@ import com.yahoo.searchdefinition.document.SDField; */ public interface FieldOperation extends Comparable { + /** Apply this operation on the given field */ void apply(SDField field); @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index af04deb5347..d8b95391ecb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -16,7 +16,7 @@ import java.util.HashSet; import java.util.Set; /** - * @author Simon Thoresen + * @author Simon Thoresen */ public class IndexingValidation extends Processor { @@ -26,11 +26,11 @@ public class IndexingValidation extends Processor { @Override public void process() { - VerificationContext ctx = new VerificationContext(new MyAdapter(search)); + VerificationContext context = new VerificationContext(new MyAdapter(search)); for (SDField field : search.allFieldsList()) { ScriptExpression script = field.getIndexingScript(); try { - script.verify(ctx); + script.verify(context); MyConverter converter = new MyConverter(); for (StatementExpression exp : script) { converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch @@ -123,8 +123,7 @@ public class IndexingValidation extends Processor { throw new UnsupportedOperationException(); } if (!fieldType.isAssignableFrom(valueType) && - !fieldType.isAssignableFrom(createCompatType(valueType))) - { + !fieldType.isAssignableFrom(createCompatType(valueType))) { throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + " '" + fieldName + "' which is " + fieldType.getName() + "."); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index ae16f6cfed8..36cf302477e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -2,7 +2,9 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; @@ -23,12 +25,13 @@ public class TensorFieldProcessor extends Processor { @Override public void process() { for (SDField field : search.allFieldsList()) { - if (field.getDataType() == DataType.TENSOR) { + if ( field.getDataType() instanceof TensorDataType ) { warnUseOfTensorFieldAsAttribute(field); validateIndexingScripsForTensorField(field); validateAttributeSettingForTensorField(field); - } else { - validateDataTypeForField(field); + } + else if (field.getDataType() instanceof CollectionDataType){ + validateDataTypeForCollectionField(field); } } } @@ -55,9 +58,9 @@ public class TensorFieldProcessor extends Processor { } } - private void validateDataTypeForField(SDField field) { - if (field.getDataType().getPrimitiveType() == DataType.TENSOR) { + private void validateDataTypeForCollectionField(SDField field) { + if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); - } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 1349abc3795..9ee5f48a906 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.document.DataType; @@ -49,22 +50,25 @@ public class ValidateFieldTypes extends Processor { DataType seenType = seenFields.get(fieldName); if (seenType == null) { seenFields.put(fieldName, fieldType); - } else if ( ! equalTypes(seenType, fieldType)) { - throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + + } else if ( ! compatibleTypes(seenType, fieldType)) { + throw newProcessException(searchName, fieldName, "Incompatible types. Expected " + seenType.getName() + " for " + fieldDesc + " '" + fieldName + "', got " + fieldType.getName() + "."); } } - private boolean equalTypes(DataType d1, DataType d2) { + private boolean compatibleTypes(DataType seenType, DataType fieldType) { // legacy tag field type compatibility; probably not needed any more (Oct 2016) - if ("tag".equals(d1.getName())) { - return "tag".equals(d2.getName()) || "WeightedSet".equals(d2.getName()); + if ("tag".equals(seenType.getName())) { + return "tag".equals(fieldType.getName()) || "WeightedSet".equals(fieldType.getName()); } - if ("tag".equals(d2.getName())) { - return "tag".equals(d1.getName()) || "WeightedSet".equals(d1.getName()); + if ("tag".equals(fieldType.getName())) { + return "tag".equals(seenType.getName()) || "WeightedSet".equals(seenType.getName()); } - return d1.equals(d2); + if (seenType instanceof TensorDataType && fieldType instanceof TensorDataType) { + return fieldType.isAssignableFrom(seenType); // TODO: Just do this for all types + } + return seenType.equals(fieldType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 3cdcf55b34e..705ec4dc4a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -17,39 +17,39 @@ import java.util.Set; /** * @author baldersheim - * @since 2010-02-19 */ public class DocumentManager { - public DocumentmanagerConfig.Builder produce(DocumentModel model, DocumentmanagerConfig.Builder docman) { - docman.enablecompression(false); + public DocumentmanagerConfig.Builder produce(DocumentModel model, + DocumentmanagerConfig.Builder documentConfigBuilder) { + documentConfigBuilder.enablecompression(false); Set handled = new HashSet<>(); for(NewDocumentType documentType : model.getDocumentManager().getTypes()) { - handle(documentType, docman, handled); - handleAnnotations(documentType.getAnnotations(), docman); + buildConfig(documentType, documentConfigBuilder, handled); + buildConfig(documentType.getAnnotations(), documentConfigBuilder); if ( documentType != VespaDocumentType.INSTANCE) { - DocumentmanagerConfig.Datatype.Builder dt = new DocumentmanagerConfig.Datatype.Builder(); - docman.datatype(dt); - handleDataType(documentType, dt); + DocumentmanagerConfig.Datatype.Builder dataTypeBuilder = new DocumentmanagerConfig.Datatype.Builder(); + documentConfigBuilder.datatype(dataTypeBuilder); + buildConfig(documentType, dataTypeBuilder); } } - return docman; + return documentConfigBuilder; } - private void handle(DataTypeCollection type, DocumentmanagerConfig.Builder docman, Set handled) { + private void buildConfig(DataTypeCollection type, DocumentmanagerConfig.Builder documentConfigBuilder, Set built) { for (DataType dataType : type.getTypes()) { - if (handled.contains(dataType)) continue; - handled.add(dataType); + if (built.contains(dataType)) continue; + built.add(dataType); if (dataType instanceof TemporaryStructuredDataType) continue; - if ((dataType.getId() < 0) || (DataType.lastPredefinedDataTypeId() < dataType.getId())) { - Datatype.Builder dtc = new Datatype.Builder(); - docman.datatype(dtc); - handleDataType(dataType, dtc); + if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) { + Datatype.Builder dataTypeBuilder = new Datatype.Builder(); + documentConfigBuilder.datatype(dataTypeBuilder); + buildConfig(dataType, dataTypeBuilder); } } } - private void handleAnnotation(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) { + private void buildConfig(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) { atb. id(type.getId()). name(type.getName()); @@ -62,34 +62,34 @@ public class DocumentManager { } } } - private void handleAnnotations(Collection types, DocumentmanagerConfig.Builder builder) { + private void buildConfig(Collection types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); - handleAnnotation(type, atb); + buildConfig(type, atb); builder.annotationtype(atb); } } - private void handleDataType(DataType type, Datatype.Builder dtc) { - dtc.id(type.getId()); + private void buildConfig(DataType type, Datatype.Builder builder) { + builder.id(type.getId()); if (type instanceof ArrayDataType) { CollectionDataType dt = (CollectionDataType) type; - dtc.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId())); + builder.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId())); } else if (type instanceof WeightedSetDataType) { WeightedSetDataType dt = (WeightedSetDataType) type; - dtc.weightedsettype(new Datatype.Weightedsettype.Builder(). + builder.weightedsettype(new Datatype.Weightedsettype.Builder(). datatype(dt.getNestedType().getId()). createifnonexistant(dt.createIfNonExistent()). removeifzero(dt.removeIfZero())); } else if (type instanceof MapDataType) { MapDataType mtype = (MapDataType) type; - dtc.maptype(new Datatype.Maptype.Builder(). + builder.maptype(new Datatype.Maptype.Builder(). keytype(mtype.getKeyType().getId()). valtype(mtype.getValueType().getId())); } else if (type instanceof DocumentType) { DocumentType dt = (DocumentType) type; Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder(); - dtc.documenttype(doc); + builder.documenttype(doc); doc. name(dt.getName()). headerstruct(dt.getHeaderType().getId()). @@ -100,7 +100,7 @@ public class DocumentManager { } else if (type instanceof NewDocumentType) { NewDocumentType dt = (NewDocumentType) type; Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder(); - dtc.documenttype(doc); + builder.documenttype(doc); doc. name(dt.getName()). headerstruct(dt.getHeader().getId()). @@ -108,49 +108,56 @@ public class DocumentManager { for (NewDocumentType inherited : dt.getInherited()) { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } - handleFieldSets(dt.getFieldSets(), doc); + buildConfig(dt.getFieldSets(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { - StructDataType dt = (StructDataType) type; - Datatype.Structtype.Builder st = new Datatype.Structtype.Builder(); - dtc.structtype(st); - st.name(dt.getName()); - if (dt.getCompressionConfig().type.getCode() != 0) { - st. - compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(dt.getCompressionConfig().type.toString())). - compresslevel(dt.getCompressionConfig().compressionLevel). - compressthreshold((int)dt.getCompressionConfig().threshold). - compressminsize((int)dt.getCompressionConfig().minsize); + StructDataType structType = (StructDataType) type; + Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder(); + builder.structtype(structBuilder); + structBuilder.name(structType.getName()); + if (structType.getCompressionConfig().type.getCode() != 0) { + structBuilder. + compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(structType.getCompressionConfig().type.toString())). + compresslevel(structType.getCompressionConfig().compressionLevel). + compressthreshold((int)structType.getCompressionConfig().threshold). + compressminsize((int)structType.getCompressionConfig().minsize); } - for (com.yahoo.document.Field field : dt.getFieldsThisTypeOnly()) { - Datatype.Structtype.Field.Builder fb = new Datatype.Structtype.Field.Builder(); - st.field(fb); - fb.name(field.getName()); + for (com.yahoo.document.Field field : structType.getFieldsThisTypeOnly()) { + Datatype.Structtype.Field.Builder fieldBuilder = new Datatype.Structtype.Field.Builder(); + structBuilder.field(fieldBuilder); + fieldBuilder.name(field.getName()); if (field.hasForcedId()) { - fb.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId())); + fieldBuilder.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId())); } - fb.datatype(field.getDataType().getId()); + fieldBuilder.datatype(field.getDataType().getId()); + + if (field.getDataType() instanceof TensorDataType) + fieldBuilder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString()); } - for (StructDataType inherited : dt.getInheritedTypes()) { - st.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName())); + for (StructDataType inherited : structType.getInheritedTypes()) { + structBuilder.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName())); } } else if (type instanceof AnnotationReferenceDataType) { AnnotationReferenceDataType annotationRef = (AnnotationReferenceDataType) type; - dtc.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName())); + builder.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName())); + } else if (type instanceof TensorDataType) { + // Nothing to do; the type of the tensor is instead stored in each field as detailed type information + // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways) + // without changing the field type and thus requiring data refeed } else { - throw new IllegalArgumentException("Can not handle datatype '" + type.getName()); + throw new IllegalArgumentException("Can not create config for data type '" + type.getName()); } } - private void handleFieldSets(Set
fieldSets, Datatype.Documenttype.Builder doc) { - + private void buildConfig(Set
fieldSets, Datatype.Documenttype.Builder doc) { for (FieldSet builtinFs : fieldSets) { - handleFieldSet(builtinFs, doc); + buildConfig(builtinFs, doc); } } - private void handleFieldSet(FieldSet fs, Datatype.Documenttype.Builder doc) { + private void buildConfig(FieldSet fs, Datatype.Documenttype.Builder doc) { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 2ba34def464..ebe18b36410 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.configmodel.producers; import com.yahoo.document.*; -import com.yahoo.document.DocumenttypesConfig.Builder; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.FieldSets; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; import java.util.*; @@ -31,12 +29,12 @@ public class DocumentTypes { for (NewDocumentType inherited : documentType.getInherited()) { produceInheritOrder(inherited, builder, produced); } - handle(documentType, builder); + buildConfig(documentType, builder); produced.put(documentType.getFullName(), documentType); } } - private void handle(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { + private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { if (documentType == VespaDocumentType.INSTANCE) { return; } @@ -46,40 +44,40 @@ public class DocumentTypes { name(documentType.getName()). headerstruct(documentType.getHeader().getId()). bodystruct(documentType.getBody().getId()); - Set handled = new HashSet<>(); + Set built = new HashSet<>(); for (NewDocumentType inherited : documentType.getInherited()) { db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); - markAsHandled(handled, inherited.getAllTypes()); + markAsBuilt(built, inherited.getAllTypes()); } for (DataType dt : documentType.getTypes()) { - handle(dt, db, handled); + buildConfig(dt, db, built); } for(AnnotationType annotation : documentType.getAnnotations()) { DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder(); db.annotationtype(atb); - handle(annotation, atb); + buildConfig(annotation, atb); } - handleFieldSets(documentType.getFieldSets(), db); + buildConfig(documentType.getFieldSets(), db); builder.documenttype(db); } - private void handleFieldSets(Set
fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) { + private void buildConfig(Set
fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) { for (FieldSet fs : fieldSets) { - handleFieldSet(fs, db); + buildConfig(fs, db); } } - private void handleFieldSet(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) { + private void buildConfig(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) { db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } - private void markAsHandled(Set handled, DataTypeCollection typeCollection) { + private void markAsBuilt(Set built, DataTypeCollection typeCollection) { for (DataType type : typeCollection.getTypes()) { - handled.add(type.getId()); + built.add(type.getId()); } } - private void handle(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) { + private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) { builder. id(annotation.getId()). name(annotation.getName()); @@ -92,67 +90,77 @@ public class DocumentTypes { } } - private void handle(DataType type, DocumenttypesConfig.Documenttype.Builder db, Set handled) { - if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! handled.contains(type.getId())) { - handled.add(type.getId()); - DocumenttypesConfig.Documenttype.Datatype.Builder dtb = new DocumenttypesConfig.Documenttype.Datatype.Builder(); - dtb.id(type.getId()); + private void buildConfig(DataType type, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set built) { + if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! built.contains(type.getId())) { + built.add(type.getId()); + DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder(); + dataTypeBuilder.id(type.getId()); if (type instanceof StructDataType) { - dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("STRUCT")); + dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT); StructDataType dt = (StructDataType) type; - DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder sb = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); - dtb.sstruct(sb); - sb.name(dt.getName()); + DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); + dataTypeBuilder.sstruct(structBuilder); + structBuilder.name(dt.getName()); if (dt.getCompressionConfig().type.getCode() != 0) { - sb.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). + structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(dt.getCompressionConfig().type.toString())). level(dt.getCompressionConfig().compressionLevel). threshold((int)dt.getCompressionConfig().threshold). minsize((int)dt.getCompressionConfig().minsize)); } for (com.yahoo.document.Field field : dt.getFields()) { - sb.field(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(). - name(field.getName()). + DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder = + new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(); + builder.name(field.getName()). id(field.getId()). id_v6(field.getIdV6()). - datatype(field.getDataType().getId())); - handle(field.getDataType(), db, handled); + datatype(field.getDataType().getId()); + if (field.getDataType() instanceof TensorDataType) + builder.detailedtype(((TensorDataType)field.getDataType()).getTensorType().toString()); + structBuilder.field(builder); + buildConfig(field.getDataType(), documentBuilder, built); } } else if (type instanceof ArrayDataType) { - dtb. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ARRAY")). + dataTypeBuilder. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY). array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder(). element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(((ArrayDataType)type).getNestedType().getId()))); - handle(((ArrayDataType)type).getNestedType(), db, handled); + buildConfig(((ArrayDataType)type).getNestedType(), documentBuilder, built); } else if (type instanceof WeightedSetDataType) { - dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("WSET")). + dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET). wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder(). key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder(). id(((WeightedSetDataType)type).getNestedType().getId())). createifnonexistent(((WeightedSetDataType)type).createIfNonExistent()). removeifzero(((WeightedSetDataType)type).removeIfZero())); - handle(((WeightedSetDataType)type).getNestedType(), db, handled); + buildConfig(((WeightedSetDataType)type).getNestedType(), documentBuilder, built); } else if (type instanceof MapDataType) { - dtb. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("MAP")). + dataTypeBuilder. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP). map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder(). key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder(). id(((MapDataType)type).getKeyType().getId())). value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder(). id(((MapDataType)type).getValueType().getId()))); - handle(((MapDataType)type).getKeyType(), db, handled); - handle(((MapDataType)type).getValueType(), db, handled); + buildConfig(((MapDataType)type).getKeyType(), documentBuilder, built); + buildConfig(((MapDataType)type).getValueType(), documentBuilder, built); } else if (type instanceof AnnotationReferenceDataType) { - dtb. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ANNOTATIONREF")). + dataTypeBuilder. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ANNOTATIONREF). annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder(). annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder(). id(((AnnotationReferenceDataType)type).getAnnotationType().getId()))); + } else if (type instanceof TensorDataType) { + // The type of the tensor is not stored here but instead in each field as detailed type information + // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways) + // without changing the field type and thus requiring data refeed + return; } else { return; } - db.datatype(dtb); + documentBuilder.datatype(dataTypeBuilder); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index c03fb0617b8..1d39d1e6928 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -43,7 +43,7 @@ public class SearchDataTypeValidator extends Validator { for (Field field : doc.fieldSet()) { DataType fieldType = field.getDataType(); disallowIndexingOfMaps(cluster, def, field); - if (!validateDataType(fieldType)) { + if ( ! isSupportedInSearchClusters(fieldType)) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " + "clusters (field '" + field.getName() + "' in definition '" + def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); @@ -51,28 +51,30 @@ public class SearchDataTypeValidator extends Validator { } } - private boolean validateDataType(DataType dataType) { - if (dataType instanceof ArrayDataType || - dataType instanceof WeightedSetDataType) - { - return validateDataType(((CollectionDataType)dataType).getNestedType()); + private boolean isSupportedInSearchClusters(DataType dataType) { + if (dataType instanceof ArrayDataType || dataType instanceof WeightedSetDataType) { + return isSupportedInSearchClusters(((CollectionDataType)dataType).getNestedType()); } - if (dataType instanceof StructDataType) { + else if (dataType instanceof StructDataType) { return true; // Struct will work for summary TODO maybe check individual fields } - if (dataType instanceof MapDataType) { + else if (dataType instanceof MapDataType) { return true; // Maps will work for summary, see disallowIndexingOfMaps() } - return dataType.equals(DataType.INT) || - dataType.equals(DataType.FLOAT) || - dataType.equals(DataType.STRING) || - dataType.equals(DataType.RAW) || - dataType.equals(DataType.LONG) || - dataType.equals(DataType.DOUBLE) || - dataType.equals(DataType.URI) || - dataType.equals(DataType.BYTE) || - dataType.equals(DataType.PREDICATE) || - dataType.equals(DataType.TENSOR); + else if (dataType instanceof TensorDataType) { + return true; + } + else { + return dataType.equals(DataType.INT) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.STRING) || + dataType.equals(DataType.RAW) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.URI) || + dataType.equals(DataType.BYTE) || + dataType.equals(DataType.PREDICATE); + } } private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 32be8906ffc..3fc84f2860b 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -805,13 +805,14 @@ DataType dataType() : DataType mapType=null; DataType arrayType=null; DataType wsetType=null; - + TensorType tensorType; } { ( LOOKAHEAD( ) ( arrayType = dataType() { return DataType.getArray(arrayType); } ) | LOOKAHEAD( ) ( wsetType = dataType() { return DataType.getWeightedSet(wsetType); } ) | LOOKAHEAD( ) ( mapType = mapDataType() { return mapType; } ) | LOOKAHEAD( ) ( mapType = annotationRefDataType() { return mapType; } ) + | LOOKAHEAD() ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } ) | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] ) ) { diff --git a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd index d2851816bc0..a2b36416fd6 100644 --- a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd @@ -11,6 +11,6 @@ search simple { field my_uri type uri { indexing: summary } field my_byte type byte { indexing: summary } field my_predicate type predicate { indexing: summary } - field my_tensor type tensor { indexing: summary } + field my_tensor type tensor(x{}) { indexing: summary } } } diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index ac148209d6b..6b01934307a 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1865479609 datatype[1].maptype[0].keytype 2 datatype[1].maptype[0].valtype 4 @@ -22,12 +24,16 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "Version" datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "Name" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "FlagsCounter" datatype[2].structtype[0].field[2].datatype -1865479609 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "anotherfolder" datatype[2].structtype[0].field[3].datatype 294108848 +datatype[2].structtype[0].field[3].detailedtype "" datatype[3].id 109267174 datatype[3].structtype[0].name "sct" datatype[3].structtype[0].version 0 @@ -37,8 +43,10 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "s1" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "s2" datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" datatype[4].id 49942803 datatype[4].arraytype[0].datatype 16 datatype[5].id 339965458 @@ -53,12 +61,16 @@ datatype[6].structtype[0].compressthreshold 95 datatype[6].structtype[0].compressminsize 800 datatype[6].structtype[0].field[0].name "bytearr" datatype[6].structtype[0].field[0].datatype 49942803 +datatype[6].structtype[0].field[0].detailedtype "" datatype[6].structtype[0].field[1].name "mymap" datatype[6].structtype[0].field[1].datatype 339965458 +datatype[6].structtype[0].field[1].detailedtype "" datatype[6].structtype[0].field[2].name "title" datatype[6].structtype[0].field[2].datatype 2 +datatype[6].structtype[0].field[2].detailedtype "" datatype[6].structtype[0].field[3].name "structfield" datatype[6].structtype[0].field[3].datatype 2 +datatype[6].structtype[0].field[3].detailedtype "" datatype[7].id -1245117006 datatype[7].arraytype[0].datatype 0 datatype[8].id 1328286588 @@ -115,62 +127,91 @@ datatype[24].structtype[0].compressthreshold 95 datatype[24].structtype[0].compressminsize 800 datatype[24].structtype[0].field[0].name "abyte" datatype[24].structtype[0].field[0].datatype 16 +datatype[24].structtype[0].field[0].detailedtype "" datatype[24].structtype[0].field[1].name "along" datatype[24].structtype[0].field[1].datatype 4 +datatype[24].structtype[0].field[1].detailedtype "" datatype[24].structtype[0].field[2].name "arrayfield" datatype[24].structtype[0].field[2].datatype -1245117006 +datatype[24].structtype[0].field[2].detailedtype "" datatype[24].structtype[0].field[3].name "setfield" datatype[24].structtype[0].field[3].datatype 1328286588 +datatype[24].structtype[0].field[3].detailedtype "" datatype[24].structtype[0].field[4].name "pos" datatype[24].structtype[0].field[4].datatype 1381038251 +datatype[24].structtype[0].field[4].detailedtype "" datatype[24].structtype[0].field[5].name "setfield2" datatype[24].structtype[0].field[5].datatype 18 +datatype[24].structtype[0].field[5].detailedtype "" datatype[24].structtype[0].field[6].name "setfield3" datatype[24].structtype[0].field[6].datatype 2125328771 +datatype[24].structtype[0].field[6].detailedtype "" datatype[24].structtype[0].field[7].name "setfield4" datatype[24].structtype[0].field[7].datatype 2065577986 +datatype[24].structtype[0].field[7].detailedtype "" datatype[24].structtype[0].field[8].name "tagfield" datatype[24].structtype[0].field[8].datatype 18 +datatype[24].structtype[0].field[8].detailedtype "" datatype[24].structtype[0].field[9].name "structfield" datatype[24].structtype[0].field[9].datatype 109267174 +datatype[24].structtype[0].field[9].detailedtype "" datatype[24].structtype[0].field[10].name "structarrayfield" datatype[24].structtype[0].field[10].datatype -1244829667 +datatype[24].structtype[0].field[10].detailedtype "" datatype[24].structtype[0].field[11].name "stringmapfield" datatype[24].structtype[0].field[11].datatype 339965458 +datatype[24].structtype[0].field[11].detailedtype "" datatype[24].structtype[0].field[12].name "intmapfield" datatype[24].structtype[0].field[12].datatype -1584287606 +datatype[24].structtype[0].field[12].detailedtype "" datatype[24].structtype[0].field[13].name "floatmapfield" datatype[24].structtype[0].field[13].datatype 2125154557 +datatype[24].structtype[0].field[13].detailedtype "" datatype[24].structtype[0].field[14].name "longmapfield" datatype[24].structtype[0].field[14].datatype -1715531035 +datatype[24].structtype[0].field[14].detailedtype "" datatype[24].structtype[0].field[15].name "doublemapfield" datatype[24].structtype[0].field[15].datatype 2138385264 +datatype[24].structtype[0].field[15].detailedtype "" datatype[24].structtype[0].field[16].name "arraymapfield" datatype[24].structtype[0].field[16].datatype 435886609 +datatype[24].structtype[0].field[16].detailedtype "" datatype[24].structtype[0].field[17].name "arrarr" datatype[24].structtype[0].field[17].datatype -794985308 +datatype[24].structtype[0].field[17].detailedtype "" datatype[24].structtype[0].field[18].name "maparr" datatype[24].structtype[0].field[18].datatype 69621385 +datatype[24].structtype[0].field[18].detailedtype "" datatype[24].structtype[0].field[19].name "mystructfield" datatype[24].structtype[0].field[19].datatype -2092985853 +datatype[24].structtype[0].field[19].detailedtype "" datatype[24].structtype[0].field[20].name "mystructmap" datatype[24].structtype[0].field[20].datatype 1901258752 +datatype[24].structtype[0].field[20].detailedtype "" datatype[24].structtype[0].field[21].name "mystructarr" datatype[24].structtype[0].field[21].datatype 759956026 +datatype[24].structtype[0].field[21].detailedtype "" datatype[24].structtype[0].field[22].name "Folders" datatype[24].structtype[0].field[22].datatype -389833101 +datatype[24].structtype[0].field[22].detailedtype "" datatype[24].structtype[0].field[23].name "juletre" datatype[24].structtype[0].field[23].datatype 4 +datatype[24].structtype[0].field[23].detailedtype "" datatype[24].structtype[0].field[24].name "album0" datatype[24].structtype[0].field[24].datatype 18 +datatype[24].structtype[0].field[24].detailedtype "" datatype[24].structtype[0].field[25].name "album1" datatype[24].structtype[0].field[25].datatype 18 +datatype[24].structtype[0].field[25].detailedtype "" datatype[24].structtype[0].field[26].name "other" datatype[24].structtype[0].field[26].datatype 4 +datatype[24].structtype[0].field[26].detailedtype "" datatype[24].structtype[0].field[27].name "rankfeatures" datatype[24].structtype[0].field[27].datatype 2 +datatype[24].structtype[0].field[27].detailedtype "" datatype[24].structtype[0].field[28].name "summaryfeatures" datatype[24].structtype[0].field[28].datatype 2 +datatype[24].structtype[0].field[28].detailedtype "" datatype[25].id 171503364 datatype[25].maptype[0].keytype 1707615575 datatype[25].maptype[0].valtype 0 @@ -185,6 +226,7 @@ datatype[27].structtype[0].compressthreshold 95 datatype[27].structtype[0].compressminsize 800 datatype[27].structtype[0].field[0].name "complexarray" datatype[27].structtype[0].field[0].datatype 1100964733 +datatype[27].structtype[0].field[0].detailedtype "" datatype[28].id -853072901 datatype[28].documenttype[0].name "types" datatype[28].documenttype[0].version 0 @@ -218,4 +260,4 @@ datatype[28].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" datatype[28].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" datatype[28].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" datatype[28].documenttype[0].fieldsets{[document]}.fields[25] "structfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" \ No newline at end of file diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index eacd878e13d..dc7962adec7 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -39,18 +39,22 @@ documenttype[0].datatype[1].sstruct.field[0].name "Version" documenttype[0].datatype[1].sstruct.field[0].id 64430502 documenttype[0].datatype[1].sstruct.field[0].id_v6 634243672 documenttype[0].datatype[1].sstruct.field[0].datatype 0 +documenttype[0].datatype[1].sstruct.field[0].detailedtype "" documenttype[0].datatype[1].sstruct.field[1].name "Name" documenttype[0].datatype[1].sstruct.field[1].id 2002760220 documenttype[0].datatype[1].sstruct.field[1].id_v6 62942997 documenttype[0].datatype[1].sstruct.field[1].datatype 2 +documenttype[0].datatype[1].sstruct.field[1].detailedtype "" documenttype[0].datatype[1].sstruct.field[2].name "FlagsCounter" documenttype[0].datatype[1].sstruct.field[2].id 1741227606 documenttype[0].datatype[1].sstruct.field[2].id_v6 1287497652 documenttype[0].datatype[1].sstruct.field[2].datatype -1865479609 +documenttype[0].datatype[1].sstruct.field[2].detailedtype "" documenttype[0].datatype[1].sstruct.field[3].name "anotherfolder" documenttype[0].datatype[1].sstruct.field[3].id 1582421848 documenttype[0].datatype[1].sstruct.field[3].id_v6 1898725199 documenttype[0].datatype[1].sstruct.field[3].datatype 294108848 +documenttype[0].datatype[1].sstruct.field[3].detailedtype "" documenttype[0].datatype[2].id 109267174 documenttype[0].datatype[2].type STRUCT documenttype[0].datatype[2].array.element.id 0 @@ -70,10 +74,12 @@ documenttype[0].datatype[2].sstruct.field[0].name "s1" documenttype[0].datatype[2].sstruct.field[0].id 2146820765 documenttype[0].datatype[2].sstruct.field[0].id_v6 142373281 documenttype[0].datatype[2].sstruct.field[0].datatype 2 +documenttype[0].datatype[2].sstruct.field[0].detailedtype "" documenttype[0].datatype[2].sstruct.field[1].name "s2" documenttype[0].datatype[2].sstruct.field[1].id 45366795 documenttype[0].datatype[2].sstruct.field[1].id_v6 31106270 documenttype[0].datatype[2].sstruct.field[1].datatype 2 +documenttype[0].datatype[2].sstruct.field[1].detailedtype "" documenttype[0].datatype[3].id 49942803 documenttype[0].datatype[3].type ARRAY documenttype[0].datatype[3].array.element.id 16 @@ -123,18 +129,22 @@ documenttype[0].datatype[5].sstruct.field[0].name "bytearr" documenttype[0].datatype[5].sstruct.field[0].id 1079701754 documenttype[0].datatype[5].sstruct.field[0].id_v6 1198855694 documenttype[0].datatype[5].sstruct.field[0].datatype 49942803 +documenttype[0].datatype[5].sstruct.field[0].detailedtype "" documenttype[0].datatype[5].sstruct.field[1].name "mymap" documenttype[0].datatype[5].sstruct.field[1].id 1954178122 documenttype[0].datatype[5].sstruct.field[1].id_v6 707189723 documenttype[0].datatype[5].sstruct.field[1].datatype 339965458 +documenttype[0].datatype[5].sstruct.field[1].detailedtype "" documenttype[0].datatype[5].sstruct.field[2].name "title" documenttype[0].datatype[5].sstruct.field[2].id 567626448 documenttype[0].datatype[5].sstruct.field[2].id_v6 29129762 documenttype[0].datatype[5].sstruct.field[2].datatype 2 +documenttype[0].datatype[5].sstruct.field[2].detailedtype "" documenttype[0].datatype[5].sstruct.field[3].name "structfield" documenttype[0].datatype[5].sstruct.field[3].id 1726890940 documenttype[0].datatype[5].sstruct.field[3].id_v6 418303145 documenttype[0].datatype[5].sstruct.field[3].datatype 2 +documenttype[0].datatype[5].sstruct.field[3].detailedtype "" documenttype[0].datatype[6].id -1245117006 documenttype[0].datatype[6].type ARRAY documenttype[0].datatype[6].array.element.id 0 @@ -409,118 +419,147 @@ documenttype[0].datatype[23].sstruct.field[0].name "abyte" documenttype[0].datatype[23].sstruct.field[0].id 110138156 documenttype[0].datatype[23].sstruct.field[0].id_v6 1369099343 documenttype[0].datatype[23].sstruct.field[0].datatype 16 +documenttype[0].datatype[23].sstruct.field[0].detailedtype "" documenttype[0].datatype[23].sstruct.field[1].name "along" documenttype[0].datatype[23].sstruct.field[1].id 1206464520 documenttype[0].datatype[23].sstruct.field[1].id_v6 871280609 documenttype[0].datatype[23].sstruct.field[1].datatype 4 +documenttype[0].datatype[23].sstruct.field[1].detailedtype "" documenttype[0].datatype[23].sstruct.field[2].name "arrayfield" documenttype[0].datatype[23].sstruct.field[2].id 965790107 documenttype[0].datatype[23].sstruct.field[2].id_v6 1010955705 documenttype[0].datatype[23].sstruct.field[2].datatype -1245117006 +documenttype[0].datatype[23].sstruct.field[2].detailedtype "" documenttype[0].datatype[23].sstruct.field[3].name "setfield" documenttype[0].datatype[23].sstruct.field[3].id 761581914 documenttype[0].datatype[23].sstruct.field[3].id_v6 1762943268 documenttype[0].datatype[23].sstruct.field[3].datatype 1328286588 +documenttype[0].datatype[23].sstruct.field[3].detailedtype "" documenttype[0].datatype[23].sstruct.field[4].name "pos" documenttype[0].datatype[23].sstruct.field[4].id 1041567475 documenttype[0].datatype[23].sstruct.field[4].id_v6 26353693 documenttype[0].datatype[23].sstruct.field[4].datatype 1381038251 +documenttype[0].datatype[23].sstruct.field[4].detailedtype "" documenttype[0].datatype[23].sstruct.field[5].name "setfield2" documenttype[0].datatype[23].sstruct.field[5].id 1066659198 documenttype[0].datatype[23].sstruct.field[5].id_v6 813038565 documenttype[0].datatype[23].sstruct.field[5].datatype 18 +documenttype[0].datatype[23].sstruct.field[5].detailedtype "" documenttype[0].datatype[23].sstruct.field[6].name "setfield3" documenttype[0].datatype[23].sstruct.field[6].id 1180155772 documenttype[0].datatype[23].sstruct.field[6].id_v6 1697232199 documenttype[0].datatype[23].sstruct.field[6].datatype 2125328771 +documenttype[0].datatype[23].sstruct.field[6].detailedtype "" documenttype[0].datatype[23].sstruct.field[7].name "setfield4" documenttype[0].datatype[23].sstruct.field[7].id 1254131631 documenttype[0].datatype[23].sstruct.field[7].id_v6 119755202 documenttype[0].datatype[23].sstruct.field[7].datatype 2065577986 +documenttype[0].datatype[23].sstruct.field[7].detailedtype "" documenttype[0].datatype[23].sstruct.field[8].name "tagfield" documenttype[0].datatype[23].sstruct.field[8].id 1653562069 documenttype[0].datatype[23].sstruct.field[8].id_v6 938523246 documenttype[0].datatype[23].sstruct.field[8].datatype 18 +documenttype[0].datatype[23].sstruct.field[8].detailedtype "" documenttype[0].datatype[23].sstruct.field[9].name "structfield" documenttype[0].datatype[23].sstruct.field[9].id 486207386 documenttype[0].datatype[23].sstruct.field[9].id_v6 418303145 documenttype[0].datatype[23].sstruct.field[9].datatype 109267174 +documenttype[0].datatype[23].sstruct.field[9].detailedtype "" documenttype[0].datatype[23].sstruct.field[10].name "structarrayfield" documenttype[0].datatype[23].sstruct.field[10].id 335048518 documenttype[0].datatype[23].sstruct.field[10].id_v6 607034174 documenttype[0].datatype[23].sstruct.field[10].datatype -1244829667 +documenttype[0].datatype[23].sstruct.field[10].detailedtype "" documenttype[0].datatype[23].sstruct.field[11].name "stringmapfield" documenttype[0].datatype[23].sstruct.field[11].id 117465687 documenttype[0].datatype[23].sstruct.field[11].id_v6 1492788095 documenttype[0].datatype[23].sstruct.field[11].datatype 339965458 +documenttype[0].datatype[23].sstruct.field[11].detailedtype "" documenttype[0].datatype[23].sstruct.field[12].name "intmapfield" documenttype[0].datatype[23].sstruct.field[12].id 121004462 documenttype[0].datatype[23].sstruct.field[12].id_v6 1642487905 documenttype[0].datatype[23].sstruct.field[12].datatype -1584287606 +documenttype[0].datatype[23].sstruct.field[12].detailedtype "" documenttype[0].datatype[23].sstruct.field[13].name "floatmapfield" documenttype[0].datatype[23].sstruct.field[13].id 1239120925 documenttype[0].datatype[23].sstruct.field[13].id_v6 1609437589 documenttype[0].datatype[23].sstruct.field[13].datatype 2125154557 +documenttype[0].datatype[23].sstruct.field[13].detailedtype "" documenttype[0].datatype[23].sstruct.field[14].name "longmapfield" documenttype[0].datatype[23].sstruct.field[14].id 477718745 documenttype[0].datatype[23].sstruct.field[14].id_v6 920341968 documenttype[0].datatype[23].sstruct.field[14].datatype -1715531035 +documenttype[0].datatype[23].sstruct.field[14].detailedtype "" documenttype[0].datatype[23].sstruct.field[15].name "doublemapfield" documenttype[0].datatype[23].sstruct.field[15].id 877047192 documenttype[0].datatype[23].sstruct.field[15].id_v6 957317090 documenttype[0].datatype[23].sstruct.field[15].datatype 2138385264 +documenttype[0].datatype[23].sstruct.field[15].detailedtype "" documenttype[0].datatype[23].sstruct.field[16].name "arraymapfield" documenttype[0].datatype[23].sstruct.field[16].id 1670805928 documenttype[0].datatype[23].sstruct.field[16].id_v6 1940354311 documenttype[0].datatype[23].sstruct.field[16].datatype 435886609 +documenttype[0].datatype[23].sstruct.field[16].detailedtype "" documenttype[0].datatype[23].sstruct.field[17].name "arrarr" documenttype[0].datatype[23].sstruct.field[17].id 1962567166 documenttype[0].datatype[23].sstruct.field[17].id_v6 885141301 documenttype[0].datatype[23].sstruct.field[17].datatype -794985308 +documenttype[0].datatype[23].sstruct.field[17].detailedtype "" documenttype[0].datatype[23].sstruct.field[18].name "maparr" documenttype[0].datatype[23].sstruct.field[18].id 904375219 documenttype[0].datatype[23].sstruct.field[18].id_v6 63700074 documenttype[0].datatype[23].sstruct.field[18].datatype 69621385 +documenttype[0].datatype[23].sstruct.field[18].detailedtype "" documenttype[0].datatype[23].sstruct.field[19].name "mystructfield" documenttype[0].datatype[23].sstruct.field[19].id 1348513378 documenttype[0].datatype[23].sstruct.field[19].id_v6 2033170300 documenttype[0].datatype[23].sstruct.field[19].datatype -2092985853 +documenttype[0].datatype[23].sstruct.field[19].detailedtype "" documenttype[0].datatype[23].sstruct.field[20].name "mystructmap" documenttype[0].datatype[23].sstruct.field[20].id 1511423250 documenttype[0].datatype[23].sstruct.field[20].id_v6 449602635 documenttype[0].datatype[23].sstruct.field[20].datatype 1901258752 +documenttype[0].datatype[23].sstruct.field[20].detailedtype "" documenttype[0].datatype[23].sstruct.field[21].name "mystructarr" documenttype[0].datatype[23].sstruct.field[21].id 595856991 documenttype[0].datatype[23].sstruct.field[21].id_v6 764861972 documenttype[0].datatype[23].sstruct.field[21].datatype 759956026 +documenttype[0].datatype[23].sstruct.field[21].detailedtype "" documenttype[0].datatype[23].sstruct.field[22].name "Folders" documenttype[0].datatype[23].sstruct.field[22].id 34575524 documenttype[0].datatype[23].sstruct.field[22].id_v6 280569744 documenttype[0].datatype[23].sstruct.field[22].datatype -389833101 +documenttype[0].datatype[23].sstruct.field[22].detailedtype "" documenttype[0].datatype[23].sstruct.field[23].name "juletre" documenttype[0].datatype[23].sstruct.field[23].id 1039981530 documenttype[0].datatype[23].sstruct.field[23].id_v6 2073084146 documenttype[0].datatype[23].sstruct.field[23].datatype 4 +documenttype[0].datatype[23].sstruct.field[23].detailedtype "" documenttype[0].datatype[23].sstruct.field[24].name "album0" documenttype[0].datatype[23].sstruct.field[24].id 764312262 documenttype[0].datatype[23].sstruct.field[24].id_v6 1409364160 documenttype[0].datatype[23].sstruct.field[24].datatype 18 +documenttype[0].datatype[23].sstruct.field[24].detailedtype "" documenttype[0].datatype[23].sstruct.field[25].name "album1" documenttype[0].datatype[23].sstruct.field[25].id 1967160809 documenttype[0].datatype[23].sstruct.field[25].id_v6 1833811264 documenttype[0].datatype[23].sstruct.field[25].datatype 18 +documenttype[0].datatype[23].sstruct.field[25].detailedtype "" documenttype[0].datatype[23].sstruct.field[26].name "other" documenttype[0].datatype[23].sstruct.field[26].id 2443357 documenttype[0].datatype[23].sstruct.field[26].id_v6 903806222 documenttype[0].datatype[23].sstruct.field[26].datatype 4 +documenttype[0].datatype[23].sstruct.field[26].detailedtype "" documenttype[0].datatype[23].sstruct.field[27].name "rankfeatures" documenttype[0].datatype[23].sstruct.field[27].id 1883197392 documenttype[0].datatype[23].sstruct.field[27].id_v6 699950698 documenttype[0].datatype[23].sstruct.field[27].datatype 2 +documenttype[0].datatype[23].sstruct.field[27].detailedtype "" documenttype[0].datatype[23].sstruct.field[28].name "summaryfeatures" documenttype[0].datatype[23].sstruct.field[28].id 1840337115 documenttype[0].datatype[23].sstruct.field[28].id_v6 1981648971 documenttype[0].datatype[23].sstruct.field[28].datatype 2 +documenttype[0].datatype[23].sstruct.field[28].detailedtype "" documenttype[0].datatype[24].id 171503364 documenttype[0].datatype[24].type MAP documenttype[0].datatype[24].array.element.id 0 @@ -570,6 +609,7 @@ documenttype[0].datatype[26].sstruct.field[0].name "complexarray" documenttype[0].datatype[26].sstruct.field[0].id 1028383787 documenttype[0].datatype[26].sstruct.field[0].id_v6 658530305 documenttype[0].datatype[26].sstruct.field[0].datatype 1100964733 +documenttype[0].datatype[26].sstruct.field[0].detailedtype "" documenttype[0].fieldsets{[document]}.fields[0] "Folders" documenttype[0].fieldsets{[document]}.fields[1] "abyte" documenttype[0].fieldsets{[document]}.fields[2] "album0" @@ -596,4 +636,4 @@ documenttype[0].fieldsets{[document]}.fields[22] "setfield4" documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" documenttype[0].fieldsets{[document]}.fields[25] "structfield" -documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +documenttype[0].fieldsets{[document]}.fields[26] "tagfield" \ No newline at end of file diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index d13195f1ffe..3073dd55fba 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -24,10 +24,12 @@ documenttype[0].datatype[0].sstruct.field[0].name "rankfeatures" documenttype[0].datatype[0].sstruct.field[0].id 1883197392 documenttype[0].datatype[0].sstruct.field[0].id_v6 699950698 documenttype[0].datatype[0].sstruct.field[0].datatype 2 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" documenttype[0].datatype[0].sstruct.field[1].name "summaryfeatures" documenttype[0].datatype[0].sstruct.field[1].id 1840337115 documenttype[0].datatype[0].sstruct.field[1].id_v6 1981648971 documenttype[0].datatype[0].sstruct.field[1].datatype 2 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" documenttype[0].datatype[1].id 549879017 documenttype[0].datatype[1].type STRUCT documenttype[0].datatype[1].array.element.id 0 @@ -83,14 +85,17 @@ documenttype[1].datatype[1].sstruct.field[0].name "doc_field" documenttype[1].datatype[1].sstruct.field[0].id 819293364 documenttype[1].datatype[1].sstruct.field[0].id_v6 1634907905 documenttype[1].datatype[1].sstruct.field[0].datatype -1368624373 +documenttype[1].datatype[1].sstruct.field[0].detailedtype "" documenttype[1].datatype[1].sstruct.field[1].name "rankfeatures" documenttype[1].datatype[1].sstruct.field[1].id 1883197392 documenttype[1].datatype[1].sstruct.field[1].id_v6 699950698 documenttype[1].datatype[1].sstruct.field[1].datatype 2 +documenttype[1].datatype[1].sstruct.field[1].detailedtype "" documenttype[1].datatype[1].sstruct.field[2].name "summaryfeatures" documenttype[1].datatype[1].sstruct.field[2].id 1840337115 documenttype[1].datatype[1].sstruct.field[2].id_v6 1981648971 documenttype[1].datatype[1].sstruct.field[2].datatype 2 +documenttype[1].datatype[1].sstruct.field[2].detailedtype "" documenttype[1].datatype[2].id 348447225 documenttype[1].datatype[2].type STRUCT documenttype[1].datatype[2].array.element.id 0 @@ -106,4 +111,4 @@ documenttype[1].datatype[2].sstruct.compression.type NONE documenttype[1].datatype[2].sstruct.compression.level 0 documenttype[1].datatype[2].sstruct.compression.threshold 95 documenttype[1].datatype[2].sstruct.compression.minsize 200 -documenttype[1].fieldsets{[document]}.fields[0] "doc_field" +documenttype[1].fieldsets{[document]}.fields[0] "doc_field" \ No newline at end of file diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index 98c3d379987..532f53f51d2 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1486737430 datatype[1].arraytype[0].datatype 2 datatype[2].id -1337915045 @@ -21,44 +23,64 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "debug_src" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "attributes_src" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "location_str" datatype[2].structtype[0].field[2].datatype 2 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "title_src" datatype[2].structtype[0].field[3].datatype 2 +datatype[2].structtype[0].field[3].detailedtype "" datatype[2].structtype[0].field[4].name "product_src" datatype[2].structtype[0].field[4].datatype 2 +datatype[2].structtype[0].field[4].detailedtype "" datatype[2].structtype[0].field[5].name "product2_src" datatype[2].structtype[0].field[5].datatype 2 +datatype[2].structtype[0].field[5].detailedtype "" datatype[2].structtype[0].field[6].name "product3_src" datatype[2].structtype[0].field[6].datatype 2 +datatype[2].structtype[0].field[6].detailedtype "" datatype[2].structtype[0].field[7].name "debug" datatype[2].structtype[0].field[7].datatype 2 +datatype[2].structtype[0].field[7].detailedtype "" datatype[2].structtype[0].field[8].name "attributes" datatype[2].structtype[0].field[8].datatype 2 +datatype[2].structtype[0].field[8].detailedtype "" datatype[2].structtype[0].field[9].name "title" datatype[2].structtype[0].field[9].datatype 2 +datatype[2].structtype[0].field[9].detailedtype "" datatype[2].structtype[0].field[10].name "product" datatype[2].structtype[0].field[10].datatype 2 +datatype[2].structtype[0].field[10].detailedtype "" datatype[2].structtype[0].field[11].name "product2" datatype[2].structtype[0].field[11].datatype 2 +datatype[2].structtype[0].field[11].detailedtype "" datatype[2].structtype[0].field[12].name "product3" datatype[2].structtype[0].field[12].datatype 2 +datatype[2].structtype[0].field[12].detailedtype "" datatype[2].structtype[0].field[13].name "location_zcurve" datatype[2].structtype[0].field[13].datatype 4 +datatype[2].structtype[0].field[13].detailedtype "" datatype[2].structtype[0].field[14].name "title_s" datatype[2].structtype[0].field[14].datatype 2 +datatype[2].structtype[0].field[14].detailedtype "" datatype[2].structtype[0].field[15].name "location.position" datatype[2].structtype[0].field[15].datatype -1486737430 +datatype[2].structtype[0].field[15].detailedtype "" datatype[2].structtype[0].field[16].name "location.distance" datatype[2].structtype[0].field[16].datatype 0 +datatype[2].structtype[0].field[16].detailedtype "" datatype[2].structtype[0].field[17].name "mysummary" datatype[2].structtype[0].field[17].datatype 2 +datatype[2].structtype[0].field[17].detailedtype "" datatype[2].structtype[0].field[18].name "rankfeatures" datatype[2].structtype[0].field[18].datatype 2 +datatype[2].structtype[0].field[18].detailedtype "" datatype[2].structtype[0].field[19].name "summaryfeatures" datatype[2].structtype[0].field[19].datatype 2 +datatype[2].structtype[0].field[19].detailedtype "" datatype[3].id -704605648 datatype[3].structtype[0].name "advanced.body" datatype[3].structtype[0].version 0 diff --git a/config-model/src/test/derived/advanced/index-info.cfg b/config-model/src/test/derived/advanced/index-info.cfg index 1bb68110dae..7af5b858d3a 100644 --- a/config-model/src/test/derived/advanced/index-info.cfg +++ b/config-model/src/test/derived/advanced/index-info.cfg @@ -75,23 +75,23 @@ indexinfo[0].command[36].indexname "summaryfeatures" indexinfo[0].command[36].command "index" indexinfo[0].command[37].indexname "title" indexinfo[0].command[37].command "index" -indexinfo[0].command[38].indexname "titleabstract" -indexinfo[0].command[38].command "index" -indexinfo[0].command[39].indexname "default" -indexinfo[0].command[39].command "index" -indexinfo[0].command[40].indexname "title" -indexinfo[0].command[40].command "lowercase" +indexinfo[0].command[38].indexname "title" +indexinfo[0].command[38].command "lowercase" +indexinfo[0].command[39].indexname "title" +indexinfo[0].command[39].command "normalize" +indexinfo[0].command[40].indexname "title_s" +indexinfo[0].command[40].command "index" indexinfo[0].command[41].indexname "titleabstract" indexinfo[0].command[41].command "lowercase" -indexinfo[0].command[42].indexname "default" -indexinfo[0].command[42].command "lowercase" -indexinfo[0].command[43].indexname "title" +indexinfo[0].command[42].indexname "titleabstract" +indexinfo[0].command[42].command "index" +indexinfo[0].command[43].indexname "titleabstract" indexinfo[0].command[43].command "normalize" -indexinfo[0].command[44].indexname "titleabstract" -indexinfo[0].command[44].command "normalize" +indexinfo[0].command[44].indexname "default" +indexinfo[0].command[44].command "lowercase" indexinfo[0].command[45].indexname "default" -indexinfo[0].command[45].command "normalize" -indexinfo[0].command[46].indexname "title_s" -indexinfo[0].command[46].command "index" +indexinfo[0].command[45].command "index" +indexinfo[0].command[46].indexname "default" +indexinfo[0].command[46].command "normalize" indexinfo[0].alias[0].alias "headline" indexinfo[0].alias[0].indexname "title" \ No newline at end of file diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index db85eb92e9b..1e9ffba5ca0 100755 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 517946310 datatype[1].structtype[0].name "annotation.banana" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "brand" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id -364910881 datatype[2].structtype[0].name "annotationsimplicitstruct.header" datatype[2].structtype[0].version 0 @@ -28,8 +31,10 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "rankfeatures" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "summaryfeatures" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[3].id -1503592268 datatype[3].structtype[0].name "annotationsimplicitstruct.body" datatype[3].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 11e179ec748..db7fba5d9a6 100755 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 517946310 datatype[1].structtype[0].name "annotation.banana" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "brand" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id -1339036621 datatype[2].structtype[0].name "annotation.intern" datatype[2].structtype[0].version 0 @@ -28,6 +31,7 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "enddate" datatype[2].structtype[0].field[0].datatype 4 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].inherits[0].name "annotation.employee" datatype[2].structtype[0].inherits[0].version 0 datatype[3].id 249059607 @@ -39,6 +43,7 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "color" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].inherits[0].name "annotation.vehicle" datatype[3].structtype[0].inherits[0].version 0 datatype[4].id -1466283082 @@ -50,6 +55,7 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "name" datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[0].detailedtype "" datatype[5].id -858216177 datatype[5].structtype[0].name "annotation.employee" datatype[5].structtype[0].version 0 @@ -59,6 +65,7 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "employeeid" datatype[5].structtype[0].field[0].datatype 0 +datatype[5].structtype[0].field[0].detailedtype "" datatype[5].structtype[0].inherits[0].name "annotation.worker" datatype[5].structtype[0].inherits[0].version 0 datatype[6].id -1874092641 @@ -79,6 +86,7 @@ datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 datatype[7].structtype[0].field[0].name "numwheels" datatype[7].structtype[0].field[0].datatype 0 +datatype[7].structtype[0].field[0].detailedtype "" datatype[8].id -1406250281 datatype[8].structtype[0].name "annotationsinheritance.header" datatype[8].structtype[0].version 0 @@ -88,8 +96,10 @@ datatype[8].structtype[0].compressthreshold 95 datatype[8].structtype[0].compressminsize 800 datatype[8].structtype[0].field[0].name "rankfeatures" datatype[8].structtype[0].field[0].datatype 2 +datatype[8].structtype[0].field[0].detailedtype "" datatype[8].structtype[0].field[1].name "summaryfeatures" datatype[8].structtype[0].field[1].datatype 2 +datatype[8].structtype[0].field[1].detailedtype "" datatype[9].id 1181354668 datatype[9].structtype[0].name "annotationsinheritance.body" datatype[9].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index e49d0753296..6f83fbc5ce6 100755 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1443831334 datatype[1].structtype[0].name "annotation.c" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "cfoo" datatype[1].structtype[0].field[0].datatype 0 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 1443832295 datatype[2].structtype[0].name "annotation.d" datatype[2].structtype[0].version 0 @@ -46,6 +49,7 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "gfoo" datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[0].detailedtype "" datatype[5].id 424382193 datatype[5].structtype[0].name "annotationsinheritance2.header" datatype[5].structtype[0].version 0 @@ -55,8 +59,10 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "rankfeatures" datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" datatype[5].structtype[0].field[1].name "summaryfeatures" datatype[5].structtype[0].field[1].datatype 2 +datatype[5].structtype[0].field[1].detailedtype "" datatype[6].id 1375438150 datatype[6].structtype[0].name "annotationsinheritance2.body" datatype[6].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index d612a3b168b..db333ed0a13 100755 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -2014701853 datatype[1].annotationreftype[0].annotation "super" datatype[2].id -888007918 @@ -21,6 +23,7 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "a" datatype[2].structtype[0].field[0].datatype -2014701853 +datatype[2].structtype[0].field[0].detailedtype "" datatype[3].id -1552577796 datatype[3].structtype[0].name "annotationspolymorphy.header" datatype[3].structtype[0].version 0 @@ -30,8 +33,10 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "rankfeatures" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "summaryfeatures" datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" datatype[4].id -570750959 datatype[4].structtype[0].name "annotationspolymorphy.body" datatype[4].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index 3218262b54d..25106a41836 100755 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 517946310 datatype[1].structtype[0].name "annotation.banana" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "brand" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 400622300 datatype[2].annotationreftype[0].annotation "b" datatype[3].id 1443829412 @@ -30,6 +33,7 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "foo" datatype[3].structtype[0].field[0].datatype 400622300 +datatype[3].structtype[0].field[0].detailedtype "" datatype[4].id -808460615 datatype[4].annotationreftype[0].annotation "banana" datatype[5].id -770307521 @@ -41,6 +45,7 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "what" datatype[5].structtype[0].field[0].datatype -808460615 +datatype[5].structtype[0].field[0].detailedtype "" datatype[6].id 756306917 datatype[6].annotationreftype[0].annotation "cyclic" datatype[7].id 1781099546 @@ -52,6 +57,7 @@ datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 datatype[7].structtype[0].field[0].name "blah" datatype[7].structtype[0].field[0].datatype 756306917 +datatype[7].structtype[0].field[0].detailedtype "" datatype[8].id 571255414 datatype[8].structtype[0].name "annotationsreference.header" datatype[8].structtype[0].version 0 @@ -61,8 +67,10 @@ datatype[8].structtype[0].compressthreshold 95 datatype[8].structtype[0].compressminsize 800 datatype[8].structtype[0].field[0].name "rankfeatures" datatype[8].structtype[0].field[0].datatype 2 +datatype[8].structtype[0].field[0].detailedtype "" datatype[8].structtype[0].field[1].name "summaryfeatures" datatype[8].structtype[0].field[1].datatype 2 +datatype[8].structtype[0].field[1].detailedtype "" datatype[9].id 1692909067 datatype[9].structtype[0].name "annotationsreference.body" datatype[9].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index 54832bb97f9..8908d886ff9 100755 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1205708249 datatype[1].structtype[0].name "annotationssimple.header" datatype[1].structtype[0].version 0 @@ -19,8 +21,10 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "rankfeatures" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "summaryfeatures" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[2].id -682121732 datatype[2].structtype[0].name "annotationssimple.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index f3c93b31285..ebba05e0553 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1293792650 datatype[1].structtype[0].name "my_struct" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "my_structfield" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id -1080124700 datatype[2].structtype[0].name "annotation.my_anno" datatype[2].structtype[0].version 0 @@ -28,6 +31,7 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "my_annofield" datatype[2].structtype[0].field[0].datatype 1293792650 +datatype[2].structtype[0].field[0].detailedtype "" datatype[3].id 1341437796 datatype[3].structtype[0].name "annotationsstruct.header" datatype[3].structtype[0].version 0 @@ -37,8 +41,10 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "rankfeatures" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "summaryfeatures" datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" datatype[4].id -1180029319 datatype[4].structtype[0].name "annotationsstruct.body" datatype[4].structtype[0].version 0 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 01e3945fc2a..49249a7307a 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1293792650 datatype[1].structtype[0].name "my_struct" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "my_structfield" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 754837689 datatype[2].arraytype[0].datatype 1293792650 datatype[3].id -1080124700 @@ -30,6 +33,7 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "my_annofield" datatype[3].structtype[0].field[0].datatype 754837689 +datatype[3].structtype[0].field[0].detailedtype "" datatype[4].id 94945597 datatype[4].structtype[0].name "annotationsstructarray.header" datatype[4].structtype[0].version 0 @@ -39,8 +43,10 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "rankfeatures" datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[0].detailedtype "" datatype[4].structtype[0].field[1].name "summaryfeatures" datatype[4].structtype[0].field[1].datatype 2 +datatype[4].structtype[0].field[1].detailedtype "" datatype[5].id 1616435858 datatype[5].structtype[0].name "annotationsstructarray.body" datatype[5].structtype[0].version 0 diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index 68506844b5d..f873971f931 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1486737430 datatype[1].arraytype[0].datatype 2 datatype[2].id -1245117006 @@ -27,18 +29,25 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "tags" datatype[4].structtype[0].field[0].datatype -1486737430 +datatype[4].structtype[0].field[0].detailedtype "" datatype[4].structtype[0].field[1].name "ratings" datatype[4].structtype[0].field[1].datatype -1245117006 +datatype[4].structtype[0].field[1].detailedtype "" datatype[4].structtype[0].field[2].name "a" datatype[4].structtype[0].field[2].datatype 2 +datatype[4].structtype[0].field[2].detailedtype "" datatype[4].structtype[0].field[3].name "b" datatype[4].structtype[0].field[3].datatype -1486737430 +datatype[4].structtype[0].field[3].detailedtype "" datatype[4].structtype[0].field[4].name "c" datatype[4].structtype[0].field[4].datatype 1328286588 +datatype[4].structtype[0].field[4].detailedtype "" datatype[4].structtype[0].field[5].name "rankfeatures" datatype[4].structtype[0].field[5].datatype 2 +datatype[4].structtype[0].field[5].detailedtype "" datatype[4].structtype[0].field[6].name "summaryfeatures" datatype[4].structtype[0].field[6].datatype 2 +datatype[4].structtype[0].field[6].detailedtype "" datatype[5].id -1747896808 datatype[5].structtype[0].name "arrays.body" datatype[5].structtype[0].version 0 diff --git a/config-model/src/test/derived/arrays/index-info.cfg b/config-model/src/test/derived/arrays/index-info.cfg index f5f65700d89..1439dcc8bf7 100644 --- a/config-model/src/test/derived/arrays/index-info.cfg +++ b/config-model/src/test/derived/arrays/index-info.cfg @@ -1,63 +1,63 @@ -indexinfo[].name "arrays" -indexinfo[].command[].indexname "sddocname" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "sddocname" -indexinfo[].command[].command "word" -indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "stem:SHORTEST" -indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "ratings" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "ratings" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "ratings" -indexinfo[].command[].command "attribute" -indexinfo[].command[].indexname "default" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "default" -indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" -indexinfo[].command[].indexname "default" -indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "default" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "a" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "a" -indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "a" -indexinfo[].command[].command "stem:SHORTEST" -indexinfo[].command[].indexname "a" -indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "b" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "b" -indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "b" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "b" -indexinfo[].command[].command "stem:SHORTEST" -indexinfo[].command[].indexname "b" -indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "c" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "c" -indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "c" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "c" -indexinfo[].command[].command "stem:SHORTEST" -indexinfo[].command[].indexname "c" -indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "rankfeatures" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "summaryfeatures" -indexinfo[].command[].command "index" \ No newline at end of file +indexinfo[0].name "arrays" +indexinfo[0].command[0].indexname "sddocname" +indexinfo[0].command[0].command "index" +indexinfo[0].command[1].indexname "sddocname" +indexinfo[0].command[1].command "word" +indexinfo[0].command[2].indexname "tags" +indexinfo[0].command[2].command "index" +indexinfo[0].command[3].indexname "tags" +indexinfo[0].command[3].command "lowercase" +indexinfo[0].command[4].indexname "tags" +indexinfo[0].command[4].command "multivalue" +indexinfo[0].command[5].indexname "tags" +indexinfo[0].command[5].command "stem:SHORTEST" +indexinfo[0].command[6].indexname "tags" +indexinfo[0].command[6].command "normalize" +indexinfo[0].command[7].indexname "ratings" +indexinfo[0].command[7].command "index" +indexinfo[0].command[8].indexname "ratings" +indexinfo[0].command[8].command "multivalue" +indexinfo[0].command[9].indexname "ratings" +indexinfo[0].command[9].command "attribute" +indexinfo[0].command[10].indexname "a" +indexinfo[0].command[10].command "index" +indexinfo[0].command[11].indexname "a" +indexinfo[0].command[11].command "lowercase" +indexinfo[0].command[12].indexname "a" +indexinfo[0].command[12].command "stem:SHORTEST" +indexinfo[0].command[13].indexname "a" +indexinfo[0].command[13].command "normalize" +indexinfo[0].command[14].indexname "b" +indexinfo[0].command[14].command "index" +indexinfo[0].command[15].indexname "b" +indexinfo[0].command[15].command "lowercase" +indexinfo[0].command[16].indexname "b" +indexinfo[0].command[16].command "multivalue" +indexinfo[0].command[17].indexname "b" +indexinfo[0].command[17].command "stem:SHORTEST" +indexinfo[0].command[18].indexname "b" +indexinfo[0].command[18].command "normalize" +indexinfo[0].command[19].indexname "c" +indexinfo[0].command[19].command "index" +indexinfo[0].command[20].indexname "c" +indexinfo[0].command[20].command "lowercase" +indexinfo[0].command[21].indexname "c" +indexinfo[0].command[21].command "multivalue" +indexinfo[0].command[22].indexname "c" +indexinfo[0].command[22].command "stem:SHORTEST" +indexinfo[0].command[23].indexname "c" +indexinfo[0].command[23].command "normalize" +indexinfo[0].command[24].indexname "rankfeatures" +indexinfo[0].command[24].command "index" +indexinfo[0].command[25].indexname "summaryfeatures" +indexinfo[0].command[25].command "index" +indexinfo[0].command[26].indexname "default" +indexinfo[0].command[26].command "lowercase" +indexinfo[0].command[27].indexname "default" +indexinfo[0].command[27].command "multivalue" +indexinfo[0].command[28].indexname "default" +indexinfo[0].command[28].command "index" +indexinfo[0].command[29].indexname "default" +indexinfo[0].command[29].command "stem:SHORTEST" +indexinfo[0].command[30].indexname "default" +indexinfo[0].command[30].command "normalize" \ No newline at end of file diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index 92558a62509..619bced44cf 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 49942803 datatype[1].arraytype[0].datatype 16 datatype[2].id -1068914395 @@ -55,44 +57,64 @@ datatype[13].structtype[0].compressthreshold 95 datatype[13].structtype[0].compressminsize 800 datatype[13].structtype[0].field[0].name "singlebyte" datatype[13].structtype[0].field[0].datatype 16 +datatype[13].structtype[0].field[0].detailedtype "" datatype[13].structtype[0].field[1].name "multibyte" datatype[13].structtype[0].field[1].datatype 49942803 +datatype[13].structtype[0].field[1].detailedtype "" datatype[13].structtype[0].field[2].name "wsbyte" datatype[13].structtype[0].field[2].datatype -1068914395 +datatype[13].structtype[0].field[2].detailedtype "" datatype[13].structtype[0].field[3].name "singleint" datatype[13].structtype[0].field[3].datatype 0 +datatype[13].structtype[0].field[3].detailedtype "" datatype[13].structtype[0].field[4].name "multiint" datatype[13].structtype[0].field[4].datatype -1245117006 +datatype[13].structtype[0].field[4].detailedtype "" datatype[13].structtype[0].field[5].name "wsint" datatype[13].structtype[0].field[5].datatype 519906144 +datatype[13].structtype[0].field[5].detailedtype "" datatype[13].structtype[0].field[6].name "singlelong" datatype[13].structtype[0].field[6].datatype 4 +datatype[13].structtype[0].field[6].detailedtype "" datatype[13].structtype[0].field[7].name "multilong" datatype[13].structtype[0].field[7].datatype 58874399 +datatype[13].structtype[0].field[7].detailedtype "" datatype[13].structtype[0].field[8].name "wslong" datatype[13].structtype[0].field[8].datatype -1059982799 +datatype[13].structtype[0].field[8].detailedtype "" datatype[13].structtype[0].field[9].name "singlefloat" datatype[13].structtype[0].field[9].datatype 1 +datatype[13].structtype[0].field[9].detailedtype "" datatype[13].structtype[0].field[10].name "multifloat" datatype[13].structtype[0].field[10].datatype 1650586661 +datatype[13].structtype[0].field[10].detailedtype "" datatype[13].structtype[0].field[11].name "wsfloat" datatype[13].structtype[0].field[11].datatype 1325751891 +datatype[13].structtype[0].field[11].detailedtype "" datatype[13].structtype[0].field[12].name "singledouble" datatype[13].structtype[0].field[12].datatype 5 +datatype[13].structtype[0].field[12].detailedtype "" datatype[13].structtype[0].field[13].name "multidouble" datatype[13].structtype[0].field[13].datatype -2054976470 +datatype[13].structtype[0].field[13].detailedtype "" datatype[13].structtype[0].field[14].name "wsdouble" datatype[13].structtype[0].field[14].datatype 760047548 +datatype[13].structtype[0].field[14].detailedtype "" datatype[13].structtype[0].field[15].name "singlestring" datatype[13].structtype[0].field[15].datatype 2 +datatype[13].structtype[0].field[15].detailedtype "" datatype[13].structtype[0].field[16].name "multistring" datatype[13].structtype[0].field[16].datatype -1486737430 +datatype[13].structtype[0].field[16].detailedtype "" datatype[13].structtype[0].field[17].name "wsstring" datatype[13].structtype[0].field[17].datatype 1328286588 +datatype[13].structtype[0].field[17].detailedtype "" datatype[13].structtype[0].field[18].name "rankfeatures" datatype[13].structtype[0].field[18].datatype 2 +datatype[13].structtype[0].field[18].detailedtype "" datatype[13].structtype[0].field[19].name "summaryfeatures" datatype[13].structtype[0].field[19].datatype 2 +datatype[13].structtype[0].field[19].detailedtype "" datatype[14].id 932425403 datatype[14].structtype[0].name "prefetch.body" datatype[14].structtype[0].version 0 diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index 66ebbbb2846..e47f40f100a 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1650586661 datatype[1].arraytype[0].datatype 1 datatype[2].id -1245117006 @@ -23,58 +25,85 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "title" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "location" datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" datatype[3].structtype[0].field[2].name "dyntitle" datatype[3].structtype[0].field[2].datatype 2 +datatype[3].structtype[0].field[2].detailedtype "" datatype[3].structtype[0].field[3].name "special1" datatype[3].structtype[0].field[3].datatype 2 +datatype[3].structtype[0].field[3].detailedtype "" datatype[3].structtype[0].field[4].name "special2" datatype[3].structtype[0].field[4].datatype 2 +datatype[3].structtype[0].field[4].detailedtype "" datatype[3].structtype[0].field[5].name "special3" datatype[3].structtype[0].field[5].datatype 2 +datatype[3].structtype[0].field[5].detailedtype "" datatype[3].structtype[0].field[6].name "prefixenabled" datatype[3].structtype[0].field[6].datatype 2 +datatype[3].structtype[0].field[6].detailedtype "" datatype[3].structtype[0].field[7].name "source" datatype[3].structtype[0].field[7].datatype 10 +datatype[3].structtype[0].field[7].detailedtype "" datatype[3].structtype[0].field[8].name "docurl" datatype[3].structtype[0].field[8].datatype 10 +datatype[3].structtype[0].field[8].detailedtype "" datatype[3].structtype[0].field[9].name "fleeting" datatype[3].structtype[0].field[9].datatype 1650586661 +datatype[3].structtype[0].field[9].detailedtype "" datatype[3].structtype[0].field[10].name "fleeting2" datatype[3].structtype[0].field[10].datatype 1 +datatype[3].structtype[0].field[10].detailedtype "" datatype[3].structtype[0].field[11].name "foundat" datatype[3].structtype[0].field[11].datatype 4 +datatype[3].structtype[0].field[11].detailedtype "" datatype[3].structtype[0].field[12].name "collapseby" datatype[3].structtype[0].field[12].datatype 0 +datatype[3].structtype[0].field[12].detailedtype "" datatype[3].structtype[0].field[13].name "yEaR" datatype[3].structtype[0].field[13].datatype 0 +datatype[3].structtype[0].field[13].detailedtype "" datatype[3].structtype[0].field[14].name "stringfield" datatype[3].structtype[0].field[14].datatype 2 +datatype[3].structtype[0].field[14].detailedtype "" datatype[3].structtype[0].field[15].name "exactemento" datatype[3].structtype[0].field[15].datatype 2 +datatype[3].structtype[0].field[15].detailedtype "" datatype[3].structtype[0].field[16].name "exactagain" datatype[3].structtype[0].field[16].datatype 2 +datatype[3].structtype[0].field[16].detailedtype "" datatype[3].structtype[0].field[17].name "ts" datatype[3].structtype[0].field[17].datatype 4 +datatype[3].structtype[0].field[17].detailedtype "" datatype[3].structtype[0].field[18].name "combineda" datatype[3].structtype[0].field[18].datatype 0 +datatype[3].structtype[0].field[18].detailedtype "" datatype[3].structtype[0].field[19].name "combinedb" datatype[3].structtype[0].field[19].datatype 2 +datatype[3].structtype[0].field[19].detailedtype "" datatype[3].structtype[0].field[20].name "category" datatype[3].structtype[0].field[20].datatype 2 +datatype[3].structtype[0].field[20].detailedtype "" datatype[3].structtype[0].field[21].name "woe" datatype[3].structtype[0].field[21].datatype 2 +datatype[3].structtype[0].field[21].detailedtype "" datatype[3].structtype[0].field[22].name "year_sub" datatype[3].structtype[0].field[22].datatype 0 +datatype[3].structtype[0].field[22].detailedtype "" datatype[3].structtype[0].field[23].name "year_arr" datatype[3].structtype[0].field[23].datatype -1245117006 +datatype[3].structtype[0].field[23].detailedtype "" datatype[3].structtype[0].field[24].name "exact" datatype[3].structtype[0].field[24].datatype 2 +datatype[3].structtype[0].field[24].detailedtype "" datatype[3].structtype[0].field[25].name "rankfeatures" datatype[3].structtype[0].field[25].datatype 2 +datatype[3].structtype[0].field[25].detailedtype "" datatype[3].structtype[0].field[26].name "summaryfeatures" datatype[3].structtype[0].field[26].datatype 2 +datatype[3].structtype[0].field[26].detailedtype "" datatype[4].id -1665926686 datatype[4].structtype[0].name "complex.body" datatype[4].structtype[0].version 0 diff --git a/config-model/src/test/derived/complex/rank-profiles.cfg b/config-model/src/test/derived/complex/rank-profiles.cfg index 5e2804d0ee6..a317bfeb18e 100644 --- a/config-model/src/test/derived/complex/rank-profiles.cfg +++ b/config-model/src/test/derived/complex/rank-profiles.cfg @@ -65,4 +65,4 @@ rankprofile[3].fef.property[9].value "loggrowth(1500,4000,19)" rankprofile[3].fef.property[10].name "nativeProximity.proximityTable.title" rankprofile[3].fef.property[10].value "expdecay(500,3)" rankprofile[3].fef.property[11].name "nativeProximity.reverseProximityTable.title" -rankprofile[3].fef.property[11].value "expdecay(400,3)" +rankprofile[3].fef.property[11].value "expdecay(400,3)" \ No newline at end of file diff --git a/config-model/src/test/derived/documentderiver/documentmanager.cfg b/config-model/src/test/derived/documentderiver/documentmanager.cfg index d56d90ba358..f993b465111 100644 --- a/config-model/src/test/derived/documentderiver/documentmanager.cfg +++ b/config-model/src/test/derived/documentderiver/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -843666531 datatype[1].structtype[0].name "compressed_body.header" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "from" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 1704844530 datatype[2].structtype[0].name "compressed_body.body" datatype[2].structtype[0].version 0 @@ -28,6 +31,7 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 0 datatype[2].structtype[0].field[0].name "content" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[3].id 1417245026 datatype[3].documenttype[0].name "compressed_body" datatype[3].documenttype[0].version 0 @@ -44,6 +48,7 @@ datatype[4].structtype[0].compressthreshold 50 datatype[4].structtype[0].compressminsize 0 datatype[4].structtype[0].field[0].name "from" datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[0].detailedtype "" datatype[5].id -1007627725 datatype[5].structtype[0].name "compressed_both.body" datatype[5].structtype[0].version 0 @@ -53,6 +58,7 @@ datatype[5].structtype[0].compressthreshold 90 datatype[5].structtype[0].compressminsize 0 datatype[5].structtype[0].field[0].name "content" datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" datatype[6].id 1417705345 datatype[6].documenttype[0].name "compressed_both" datatype[6].documenttype[0].version 0 @@ -69,6 +75,7 @@ datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 0 datatype[7].structtype[0].field[0].name "from" datatype[7].structtype[0].field[0].datatype 2 +datatype[7].structtype[0].field[0].detailedtype "" datatype[8].id -579052249 datatype[8].structtype[0].name "compressed_header.body" datatype[8].structtype[0].version 0 @@ -78,6 +85,7 @@ datatype[8].structtype[0].compressthreshold 95 datatype[8].structtype[0].compressminsize 800 datatype[8].structtype[0].field[0].name "content" datatype[8].structtype[0].field[0].datatype 2 +datatype[8].structtype[0].field[0].detailedtype "" datatype[9].id 1946084365 datatype[9].documenttype[0].name "compressed_header" datatype[9].documenttype[0].version 0 @@ -94,22 +102,31 @@ datatype[10].structtype[0].compressthreshold 95 datatype[10].structtype[0].compressminsize 800 datatype[10].structtype[0].field[0].name "URI" datatype[10].structtype[0].field[0].datatype 10 +datatype[10].structtype[0].field[0].detailedtype "" datatype[10].structtype[0].field[1].name "mailid" datatype[10].structtype[0].field[1].datatype 2 +datatype[10].structtype[0].field[1].detailedtype "" datatype[10].structtype[0].field[2].name "date" datatype[10].structtype[0].field[2].datatype 0 +datatype[10].structtype[0].field[2].detailedtype "" datatype[10].structtype[0].field[3].name "from" datatype[10].structtype[0].field[3].datatype 2 +datatype[10].structtype[0].field[3].detailedtype "" datatype[10].structtype[0].field[4].name "replyto" datatype[10].structtype[0].field[4].datatype 3 +datatype[10].structtype[0].field[4].detailedtype "" datatype[10].structtype[0].field[5].name "to" datatype[10].structtype[0].field[5].datatype 2 +datatype[10].structtype[0].field[5].detailedtype "" datatype[10].structtype[0].field[6].name "cc" datatype[10].structtype[0].field[6].datatype 2 +datatype[10].structtype[0].field[6].detailedtype "" datatype[10].structtype[0].field[7].name "bcc" datatype[10].structtype[0].field[7].datatype 2 +datatype[10].structtype[0].field[7].detailedtype "" datatype[10].structtype[0].field[8].name "subject" datatype[10].structtype[0].field[8].datatype 2 +datatype[10].structtype[0].field[8].detailedtype "" datatype[11].id -1244861287 datatype[11].arraytype[0].datatype 3 datatype[12].id -953584901 @@ -121,18 +138,25 @@ datatype[12].structtype[0].compressthreshold 95 datatype[12].structtype[0].compressminsize 800 datatype[12].structtype[0].field[0].name "mailbody" datatype[12].structtype[0].field[0].datatype 3 +datatype[12].structtype[0].field[0].detailedtype "" datatype[12].structtype[0].field[1].name "attachmentcount" datatype[12].structtype[0].field[1].datatype 0 +datatype[12].structtype[0].field[1].detailedtype "" datatype[12].structtype[0].field[2].name "attachmentnames" datatype[12].structtype[0].field[2].datatype 2 +datatype[12].structtype[0].field[2].detailedtype "" datatype[12].structtype[0].field[3].name "attachmenttypes" datatype[12].structtype[0].field[3].datatype 2 +datatype[12].structtype[0].field[3].detailedtype "" datatype[12].structtype[0].field[4].name "attachmentlanguages" datatype[12].structtype[0].field[4].datatype 2 +datatype[12].structtype[0].field[4].detailedtype "" datatype[12].structtype[0].field[5].name "attachmentcontent" datatype[12].structtype[0].field[5].datatype 2 +datatype[12].structtype[0].field[5].detailedtype "" datatype[12].structtype[0].field[6].name "attachments" datatype[12].structtype[0].field[6].datatype -1244861287 +datatype[12].structtype[0].field[6].detailedtype "" datatype[13].id -1081574983 datatype[13].documenttype[0].name "mail" datatype[13].documenttype[0].version 0 @@ -159,22 +183,31 @@ datatype[17].structtype[0].compressthreshold 95 datatype[17].structtype[0].compressminsize 800 datatype[17].structtype[0].field[0].name "url" datatype[17].structtype[0].field[0].datatype 10 +datatype[17].structtype[0].field[0].detailedtype "" datatype[17].structtype[0].field[1].name "title" datatype[17].structtype[0].field[1].datatype 2 +datatype[17].structtype[0].field[1].detailedtype "" datatype[17].structtype[0].field[2].name "artist" datatype[17].structtype[0].field[2].datatype 2 +datatype[17].structtype[0].field[2].detailedtype "" datatype[17].structtype[0].field[3].name "year" datatype[17].structtype[0].field[3].datatype 0 +datatype[17].structtype[0].field[3].detailedtype "" datatype[17].structtype[0].field[4].name "description" datatype[17].structtype[0].field[4].datatype 3 +datatype[17].structtype[0].field[4].detailedtype "" datatype[17].structtype[0].field[5].name "tracks" datatype[17].structtype[0].field[5].datatype -1486737430 +datatype[17].structtype[0].field[5].detailedtype "" datatype[17].structtype[0].field[6].name "popularity" datatype[17].structtype[0].field[6].datatype 519906144 +datatype[17].structtype[0].field[6].detailedtype "" datatype[17].structtype[0].field[7].name "popularity2" datatype[17].structtype[0].field[7].datatype 363959257 +datatype[17].structtype[0].field[7].detailedtype "" datatype[17].structtype[0].field[8].name "popularity3" datatype[17].structtype[0].field[8].datatype 363959257 +datatype[17].structtype[0].field[8].detailedtype "" datatype[18].id 993120973 datatype[18].structtype[0].name "music.body" datatype[18].structtype[0].version 0 @@ -198,46 +231,67 @@ datatype[20].structtype[0].compressthreshold 95 datatype[20].structtype[0].compressminsize 800 datatype[20].structtype[0].field[0].name "title" datatype[20].structtype[0].field[0].datatype 2 +datatype[20].structtype[0].field[0].detailedtype "" datatype[20].structtype[0].field[1].name "abstract" datatype[20].structtype[0].field[1].datatype 2 +datatype[20].structtype[0].field[1].detailedtype "" datatype[20].structtype[0].field[2].name "sourcename" datatype[20].structtype[0].field[2].datatype 2 +datatype[20].structtype[0].field[2].detailedtype "" datatype[20].structtype[0].field[3].name "providername" datatype[20].structtype[0].field[3].datatype 2 +datatype[20].structtype[0].field[3].detailedtype "" datatype[20].structtype[0].field[4].name "thumburl" datatype[20].structtype[0].field[4].datatype 2 +datatype[20].structtype[0].field[4].detailedtype "" datatype[20].structtype[0].field[5].name "thumbwidth" datatype[20].structtype[0].field[5].datatype 0 +datatype[20].structtype[0].field[5].detailedtype "" datatype[20].structtype[0].field[6].name "thumbheight" datatype[20].structtype[0].field[6].datatype 0 +datatype[20].structtype[0].field[6].detailedtype "" datatype[20].structtype[0].field[7].name "language" datatype[20].structtype[0].field[7].datatype 2 +datatype[20].structtype[0].field[7].detailedtype "" datatype[20].structtype[0].field[8].name "crawldocid" datatype[20].structtype[0].field[8].datatype 2 +datatype[20].structtype[0].field[8].detailedtype "" datatype[20].structtype[0].field[9].name "url" datatype[20].structtype[0].field[9].datatype 10 +datatype[20].structtype[0].field[9].detailedtype "" datatype[20].structtype[0].field[10].name "sourceurl" datatype[20].structtype[0].field[10].datatype 10 +datatype[20].structtype[0].field[10].detailedtype "" datatype[20].structtype[0].field[11].name "categories" datatype[20].structtype[0].field[11].datatype 2 +datatype[20].structtype[0].field[11].detailedtype "" datatype[20].structtype[0].field[12].name "pubdate" datatype[20].structtype[0].field[12].datatype 4 +datatype[20].structtype[0].field[12].detailedtype "" datatype[20].structtype[0].field[13].name "expdate" datatype[20].structtype[0].field[13].datatype 4 +datatype[20].structtype[0].field[13].detailedtype "" datatype[20].structtype[0].field[14].name "fingerprint" datatype[20].structtype[0].field[14].datatype 0 +datatype[20].structtype[0].field[14].detailedtype "" datatype[20].structtype[0].field[15].name "debug" datatype[20].structtype[0].field[15].datatype 2 +datatype[20].structtype[0].field[15].detailedtype "" datatype[20].structtype[0].field[16].name "attributes" datatype[20].structtype[0].field[16].datatype 2 +datatype[20].structtype[0].field[16].detailedtype "" datatype[20].structtype[0].field[17].name "searchcluster" datatype[20].structtype[0].field[17].datatype 2 +datatype[20].structtype[0].field[17].detailedtype "" datatype[20].structtype[0].field[18].name "eustaticrank" datatype[20].structtype[0].field[18].datatype 0 +datatype[20].structtype[0].field[18].detailedtype "" datatype[20].structtype[0].field[19].name "usstaticrank" datatype[20].structtype[0].field[19].datatype 0 +datatype[20].structtype[0].field[19].detailedtype "" datatype[20].structtype[0].field[20].name "asiastaticrank" datatype[20].structtype[0].field[20].datatype 0 +datatype[20].structtype[0].field[20].detailedtype "" datatype[21].id -2059783233 datatype[21].structtype[0].name "newssummary.body" datatype[21].structtype[0].version 0 @@ -261,44 +315,64 @@ datatype[23].structtype[0].compressthreshold 95 datatype[23].structtype[0].compressminsize 800 datatype[23].structtype[0].field[0].name "dynabstract" datatype[23].structtype[0].field[0].datatype 2 +datatype[23].structtype[0].field[0].detailedtype "" datatype[23].structtype[0].field[1].name "othersourcenames" datatype[23].structtype[0].field[1].datatype 2 +datatype[23].structtype[0].field[1].detailedtype "" datatype[23].structtype[0].field[2].name "author" datatype[23].structtype[0].field[2].datatype 2 +datatype[23].structtype[0].field[2].detailedtype "" datatype[23].structtype[0].field[3].name "otherlanguages" datatype[23].structtype[0].field[3].datatype 2 +datatype[23].structtype[0].field[3].detailedtype "" datatype[23].structtype[0].field[4].name "charset" datatype[23].structtype[0].field[4].datatype 2 +datatype[23].structtype[0].field[4].detailedtype "" datatype[23].structtype[0].field[5].name "mimetype" datatype[23].structtype[0].field[5].datatype 2 +datatype[23].structtype[0].field[5].detailedtype "" datatype[23].structtype[0].field[6].name "referrerurl" datatype[23].structtype[0].field[6].datatype 10 +datatype[23].structtype[0].field[6].detailedtype "" datatype[23].structtype[0].field[7].name "sourcelocation" datatype[23].structtype[0].field[7].datatype 2 +datatype[23].structtype[0].field[7].detailedtype "" datatype[23].structtype[0].field[8].name "sourcecountry" datatype[23].structtype[0].field[8].datatype 2 +datatype[23].structtype[0].field[8].detailedtype "" datatype[23].structtype[0].field[9].name "sourcelocale" datatype[23].structtype[0].field[9].datatype 2 +datatype[23].structtype[0].field[9].detailedtype "" datatype[23].structtype[0].field[10].name "sourcecontinent" datatype[23].structtype[0].field[10].datatype 2 +datatype[23].structtype[0].field[10].detailedtype "" datatype[23].structtype[0].field[11].name "articlecountry" datatype[23].structtype[0].field[11].datatype 2 +datatype[23].structtype[0].field[11].detailedtype "" datatype[23].structtype[0].field[12].name "articlelocale" datatype[23].structtype[0].field[12].datatype 2 +datatype[23].structtype[0].field[12].detailedtype "" datatype[23].structtype[0].field[13].name "articlecontinent" datatype[23].structtype[0].field[13].datatype 2 +datatype[23].structtype[0].field[13].detailedtype "" datatype[23].structtype[0].field[14].name "sourcerank" datatype[23].structtype[0].field[14].datatype 0 +datatype[23].structtype[0].field[14].detailedtype "" datatype[23].structtype[0].field[15].name "crawldate" datatype[23].structtype[0].field[15].datatype 4 +datatype[23].structtype[0].field[15].detailedtype "" datatype[23].structtype[0].field[16].name "indexdate" datatype[23].structtype[0].field[16].datatype 4 +datatype[23].structtype[0].field[16].detailedtype "" datatype[23].structtype[0].field[17].name "procdate" datatype[23].structtype[0].field[17].datatype 4 +datatype[23].structtype[0].field[17].detailedtype "" datatype[23].structtype[0].field[18].name "sourceid" datatype[23].structtype[0].field[18].datatype 0 +datatype[23].structtype[0].field[18].detailedtype "" datatype[23].structtype[0].field[19].name "sourcefeedid" datatype[23].structtype[0].field[19].datatype 0 +datatype[23].structtype[0].field[19].detailedtype "" datatype[24].id 197293167 datatype[24].structtype[0].name "newsarticle.body" datatype[24].structtype[0].version 0 @@ -308,6 +382,7 @@ datatype[24].structtype[0].compressthreshold 95 datatype[24].structtype[0].compressminsize 800 datatype[24].structtype[0].field[0].name "body" datatype[24].structtype[0].field[0].datatype 2 +datatype[24].structtype[0].field[0].detailedtype "" datatype[25].id -1710661691 datatype[25].documenttype[0].name "newsarticle" datatype[25].documenttype[0].version 0 diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index 4235199342a..9e25ff1c8ec 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 461724009 datatype[1].structtype[0].name "emptydefault.header" datatype[1].structtype[0].version 0 @@ -19,12 +21,16 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "one" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "two" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "rankfeatures" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "summaryfeatures" datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[3].detailedtype "" datatype[2].id 311791038 datatype[2].structtype[0].name "emptydefault.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index 40996d46399..00ffb00975b 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -531633022 datatype[1].structtype[0].name "id.header" datatype[1].structtype[0].version 0 @@ -19,10 +21,13 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "uri" datatype[1].structtype[0].field[0].datatype 10 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "rankfeatures" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "summaryfeatures" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[2].id -1830022377 datatype[2].structtype[0].name "id.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index 0a32e670d7b..bdc983ebb67 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -555640823 datatype[1].structtype[0].name "indexswitches.header" datatype[1].structtype[0].version 0 @@ -19,16 +21,22 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "title" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "descr" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "source_src" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "source" datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[3].detailedtype "" datatype[1].structtype[0].field[4].name "rankfeatures" datatype[1].structtype[0].field[4].datatype 2 +datatype[1].structtype[0].field[4].detailedtype "" datatype[1].structtype[0].field[5].name "summaryfeatures" datatype[1].structtype[0].field[5].datatype 2 +datatype[1].structtype[0].field[5].detailedtype "" datatype[2].id -1892617122 datatype[2].structtype[0].name "indexswitches.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/indexswitches/index-info.cfg b/config-model/src/test/derived/indexswitches/index-info.cfg index 3daf23f8b67..6387551ec98 100644 --- a/config-model/src/test/derived/indexswitches/index-info.cfg +++ b/config-model/src/test/derived/indexswitches/index-info.cfg @@ -5,37 +5,37 @@ indexinfo[0].command[1].indexname "sddocname" indexinfo[0].command[1].command "word" indexinfo[0].command[2].indexname "title" indexinfo[0].command[2].command "index" -indexinfo[0].command[3].indexname "default" -indexinfo[0].command[3].command "index" +indexinfo[0].command[3].indexname "title" +indexinfo[0].command[3].command "lowercase" indexinfo[0].command[4].indexname "title" -indexinfo[0].command[4].command "lowercase" -indexinfo[0].command[5].indexname "default" -indexinfo[0].command[5].command "lowercase" -indexinfo[0].command[6].indexname "title" -indexinfo[0].command[6].command "stem:SHORTEST" -indexinfo[0].command[7].indexname "default" -indexinfo[0].command[7].command "stem:SHORTEST" -indexinfo[0].command[8].indexname "title" -indexinfo[0].command[8].command "normalize" -indexinfo[0].command[9].indexname "default" +indexinfo[0].command[4].command "stem:SHORTEST" +indexinfo[0].command[5].indexname "title" +indexinfo[0].command[5].command "normalize" +indexinfo[0].command[6].indexname "descr" +indexinfo[0].command[6].command "index" +indexinfo[0].command[7].indexname "descr" +indexinfo[0].command[7].command "lowercase" +indexinfo[0].command[8].indexname "descr" +indexinfo[0].command[8].command "stem:SHORTEST" +indexinfo[0].command[9].indexname "descr" indexinfo[0].command[9].command "normalize" -indexinfo[0].command[10].indexname "descr" +indexinfo[0].command[10].indexname "source_src" indexinfo[0].command[10].command "index" -indexinfo[0].command[11].indexname "descr" -indexinfo[0].command[11].command "lowercase" -indexinfo[0].command[12].indexname "descr" -indexinfo[0].command[12].command "stem:SHORTEST" -indexinfo[0].command[13].indexname "descr" -indexinfo[0].command[13].command "normalize" -indexinfo[0].command[14].indexname "source_src" -indexinfo[0].command[14].command "index" -indexinfo[0].command[15].indexname "rankfeatures" +indexinfo[0].command[11].indexname "rankfeatures" +indexinfo[0].command[11].command "index" +indexinfo[0].command[12].indexname "source" +indexinfo[0].command[12].command "index" +indexinfo[0].command[13].indexname "source" +indexinfo[0].command[13].command "lowercase" +indexinfo[0].command[14].indexname "source" +indexinfo[0].command[14].command "normalize" +indexinfo[0].command[15].indexname "summaryfeatures" indexinfo[0].command[15].command "index" -indexinfo[0].command[16].indexname "source" -indexinfo[0].command[16].command "index" -indexinfo[0].command[17].indexname "source" -indexinfo[0].command[17].command "lowercase" -indexinfo[0].command[18].indexname "source" -indexinfo[0].command[18].command "normalize" -indexinfo[0].command[19].indexname "summaryfeatures" -indexinfo[0].command[19].command "index" \ No newline at end of file +indexinfo[0].command[16].indexname "default" +indexinfo[0].command[16].command "lowercase" +indexinfo[0].command[17].indexname "default" +indexinfo[0].command[17].command "index" +indexinfo[0].command[18].indexname "default" +indexinfo[0].command[18].command "stem:SHORTEST" +indexinfo[0].command[19].indexname "default" +indexinfo[0].command[19].command "normalize" \ No newline at end of file diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 8c51dc52c58..ea0aa3a3e93 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1246084544 datatype[1].structtype[0].name "grandparent_struct" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "grandparent_field" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 990971719 datatype[2].structtype[0].name "grandparent.header" datatype[2].structtype[0].version 0 @@ -28,8 +31,10 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "rankfeatures" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "summaryfeatures" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[3].id 978262812 datatype[3].structtype[0].name "grandparent.body" datatype[3].structtype[0].version 0 @@ -78,6 +83,7 @@ datatype[8].structtype[0].compressthreshold 95 datatype[8].structtype[0].compressminsize 800 datatype[8].structtype[0].field[0].name "child_field" datatype[8].structtype[0].field[0].datatype 1246084544 +datatype[8].structtype[0].field[0].detailedtype "" datatype[9].id -126593034 datatype[9].structtype[0].name "child.body" datatype[9].structtype[0].version 0 diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index fec59f5ff72..80a9cbcfc3b 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 1091188812 datatype[1].structtype[0].name "parent_struct" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "parent_field" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 836075987 datatype[2].structtype[0].name "parent.header" datatype[2].structtype[0].version 0 @@ -28,12 +31,16 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "weight_src" datatype[2].structtype[0].field[0].datatype 1 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "weight" datatype[2].structtype[0].field[1].datatype 1 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "rankfeatures" datatype[2].structtype[0].field[2].datatype 2 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "summaryfeatures" datatype[2].structtype[0].field[3].datatype 2 +datatype[2].structtype[0].field[3].detailedtype "" datatype[3].id -389494616 datatype[3].structtype[0].name "parent.body" datatype[3].structtype[0].version 0 @@ -58,6 +65,7 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "child_field" datatype[5].structtype[0].field[0].datatype 1091188812 +datatype[5].structtype[0].field[0].detailedtype "" datatype[6].id -126593034 datatype[6].structtype[0].name "child.body" datatype[6].structtype[0].version 0 diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index 44677776f3c..351ea0d9527 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -24,6 +24,7 @@ documenttype[0].datatype[0].sstruct.field[0].name "parent_field" documenttype[0].datatype[0].sstruct.field[0].id 933533022 documenttype[0].datatype[0].sstruct.field[0].id_v6 2116869443 documenttype[0].datatype[0].sstruct.field[0].datatype 2 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" documenttype[0].datatype[1].id 836075987 documenttype[0].datatype[1].type STRUCT documenttype[0].datatype[1].array.element.id 0 @@ -43,18 +44,22 @@ documenttype[0].datatype[1].sstruct.field[0].name "weight_src" documenttype[0].datatype[1].sstruct.field[0].id 1225660233 documenttype[0].datatype[1].sstruct.field[0].id_v6 1350588470 documenttype[0].datatype[1].sstruct.field[0].datatype 1 +documenttype[0].datatype[1].sstruct.field[0].detailedtype "" documenttype[0].datatype[1].sstruct.field[1].name "weight" documenttype[0].datatype[1].sstruct.field[1].id 1001392207 documenttype[0].datatype[1].sstruct.field[1].id_v6 1329620545 documenttype[0].datatype[1].sstruct.field[1].datatype 1 +documenttype[0].datatype[1].sstruct.field[1].detailedtype "" documenttype[0].datatype[1].sstruct.field[2].name "rankfeatures" documenttype[0].datatype[1].sstruct.field[2].id 1883197392 documenttype[0].datatype[1].sstruct.field[2].id_v6 699950698 documenttype[0].datatype[1].sstruct.field[2].datatype 2 +documenttype[0].datatype[1].sstruct.field[2].detailedtype "" documenttype[0].datatype[1].sstruct.field[3].name "summaryfeatures" documenttype[0].datatype[1].sstruct.field[3].id 1840337115 documenttype[0].datatype[1].sstruct.field[3].id_v6 1981648971 documenttype[0].datatype[1].sstruct.field[3].datatype 2 +documenttype[0].datatype[1].sstruct.field[3].detailedtype "" documenttype[0].datatype[2].id -389494616 documenttype[0].datatype[2].type STRUCT documenttype[0].datatype[2].array.element.id 0 @@ -97,6 +102,7 @@ documenttype[1].datatype[0].sstruct.field[0].name "child_field" documenttype[1].datatype[0].sstruct.field[0].id 1814271363 documenttype[1].datatype[0].sstruct.field[0].id_v6 405182398 documenttype[1].datatype[0].sstruct.field[0].datatype 1091188812 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" documenttype[1].datatype[1].id -126593034 documenttype[1].datatype[1].type STRUCT documenttype[1].datatype[1].array.element.id 0 diff --git a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg index 0e95d51bdd8..0a70b48fe21 100644 --- a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg +++ b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -88808602 datatype[1].structtype[0].name "mail.header" datatype[1].structtype[0].version 0 @@ -19,22 +21,31 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "URI" datatype[1].structtype[0].field[0].datatype 10 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "mailid" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "date" datatype[1].structtype[0].field[2].datatype 0 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "from" datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[3].detailedtype "" datatype[1].structtype[0].field[4].name "replyto" datatype[1].structtype[0].field[4].datatype 2 +datatype[1].structtype[0].field[4].detailedtype "" datatype[1].structtype[0].field[5].name "to" datatype[1].structtype[0].field[5].datatype 2 +datatype[1].structtype[0].field[5].detailedtype "" datatype[1].structtype[0].field[6].name "cc" datatype[1].structtype[0].field[6].datatype 2 +datatype[1].structtype[0].field[6].detailedtype "" datatype[1].structtype[0].field[7].name "bcc" datatype[1].structtype[0].field[7].datatype 2 +datatype[1].structtype[0].field[7].detailedtype "" datatype[1].structtype[0].field[8].name "subject" datatype[1].structtype[0].field[8].datatype 2 +datatype[1].structtype[0].field[8].detailedtype "" datatype[2].id -1244861287 datatype[2].arraytype[0].datatype 3 datatype[3].id -953584901 @@ -46,18 +57,25 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "body" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "attachmentcount" datatype[3].structtype[0].field[1].datatype 0 +datatype[3].structtype[0].field[1].detailedtype "" datatype[3].structtype[0].field[2].name "attachmentnames" datatype[3].structtype[0].field[2].datatype 2 +datatype[3].structtype[0].field[2].detailedtype "" datatype[3].structtype[0].field[3].name "attachmenttypes" datatype[3].structtype[0].field[3].datatype 2 +datatype[3].structtype[0].field[3].detailedtype "" datatype[3].structtype[0].field[4].name "attachmentlanguages" datatype[3].structtype[0].field[4].datatype 2 +datatype[3].structtype[0].field[4].detailedtype "" datatype[3].structtype[0].field[5].name "attachmentcontent" datatype[3].structtype[0].field[5].datatype 2 +datatype[3].structtype[0].field[5].detailedtype "" datatype[3].structtype[0].field[6].name "attachments" datatype[3].structtype[0].field[6].datatype -1244861287 +datatype[3].structtype[0].field[6].detailedtype "" datatype[4].id -1081574983 datatype[4].documenttype[0].name "mail" datatype[4].documenttype[0].version 0 diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index 1b51178e6d8..41e39ac52df 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -739138930 datatype[1].structtype[0].name "prefixexactattribute.header" datatype[1].structtype[0].version 0 @@ -19,18 +21,25 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "indexfield0" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "attributefield1" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "attributefield2" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "indexfield1" datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[3].detailedtype "" datatype[1].structtype[0].field[4].name "indexfield2" datatype[1].structtype[0].field[4].datatype 2 +datatype[1].structtype[0].field[4].detailedtype "" datatype[1].structtype[0].field[5].name "rankfeatures" datatype[1].structtype[0].field[5].datatype 2 +datatype[1].structtype[0].field[5].detailedtype "" datatype[1].structtype[0].field[6].name "summaryfeatures" datatype[1].structtype[0].field[6].datatype 2 +datatype[1].structtype[0].field[6].detailedtype "" datatype[2].id -480519133 datatype[2].structtype[0].name "prefixexactattribute.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index c57a58c244c..50be09891e5 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -471393776 datatype[1].structtype[0].name "ranktypes.header" datatype[1].structtype[0].version 0 @@ -19,18 +21,25 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "title" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "descr" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "keywords" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "identity" datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[3].detailedtype "" datatype[1].structtype[0].field[4].name "identity_literal" datatype[1].structtype[0].field[4].datatype 2 +datatype[1].structtype[0].field[4].detailedtype "" datatype[1].structtype[0].field[5].name "rankfeatures" datatype[1].structtype[0].field[5].datatype 2 +datatype[1].structtype[0].field[5].detailedtype "" datatype[1].structtype[0].field[6].name "summaryfeatures" datatype[1].structtype[0].field[6].datatype 2 +datatype[1].structtype[0].field[6].detailedtype "" datatype[2].id 1374506021 datatype[2].structtype[0].name "ranktypes.body" datatype[2].structtype[0].version 0 diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index 306f1102fb1..42826c5be33 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 105061838 datatype[1].structtype[0].name "ns1" datatype[1].structtype[0].version 0 @@ -19,10 +21,13 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "nf1" datatype[1].structtype[0].field[0].datatype 3474528 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "nf1s" datatype[1].structtype[0].field[1].datatype 3474528 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "nf2" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[2].id 3474528 datatype[2].structtype[0].name "s1" datatype[2].structtype[0].version 0 @@ -32,12 +37,16 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "f1" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "f1s" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "f2" datatype[2].structtype[0].field[2].datatype 0 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "f3" datatype[2].structtype[0].field[3].datatype 5 +datatype[2].structtype[0].field[3].detailedtype "" datatype[3].id -1497802371 datatype[3].maptype[0].keytype 4 datatype[3].maptype[0].valtype 2 @@ -52,48 +61,70 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "coupleof" datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" datatype[5].structtype[0].field[1].name "normalfields" datatype[5].structtype[0].field[1].datatype 2 +datatype[5].structtype[0].field[1].detailedtype "" datatype[5].structtype[0].field[2].name "a" datatype[5].structtype[0].field[2].datatype 3474528 +datatype[5].structtype[0].field[2].detailedtype "" datatype[5].structtype[0].field[3].name "m" datatype[5].structtype[0].field[3].datatype -1497802371 +datatype[5].structtype[0].field[3].detailedtype "" datatype[5].structtype[0].field[4].name "b" datatype[5].structtype[0].field[4].datatype 3474528 +datatype[5].structtype[0].field[4].detailedtype "" datatype[5].structtype[0].field[5].name "c" datatype[5].structtype[0].field[5].datatype 3474528 +datatype[5].structtype[0].field[5].detailedtype "" datatype[5].structtype[0].field[6].name "c2" datatype[5].structtype[0].field[6].datatype 3474528 +datatype[5].structtype[0].field[6].detailedtype "" datatype[5].structtype[0].field[7].name "c3" datatype[5].structtype[0].field[7].datatype 3474528 +datatype[5].structtype[0].field[7].detailedtype "" datatype[5].structtype[0].field[8].name "n" datatype[5].structtype[0].field[8].datatype 105061838 +datatype[5].structtype[0].field[8].detailedtype "" datatype[5].structtype[0].field[9].name "array1" datatype[5].structtype[0].field[9].datatype -1425630723 +datatype[5].structtype[0].field[9].detailedtype "" datatype[5].structtype[0].field[10].name "array2" datatype[5].structtype[0].field[10].datatype -1425630723 +datatype[5].structtype[0].field[10].detailedtype "" datatype[5].structtype[0].field[11].name "array3" datatype[5].structtype[0].field[11].datatype -1425630723 +datatype[5].structtype[0].field[11].detailedtype "" datatype[5].structtype[0].field[12].name "subject" datatype[5].structtype[0].field[12].datatype 3474528 +datatype[5].structtype[0].field[12].detailedtype "" datatype[5].structtype[0].field[13].name "d" datatype[5].structtype[0].field[13].datatype 3474528 +datatype[5].structtype[0].field[13].detailedtype "" datatype[5].structtype[0].field[14].name "e" datatype[5].structtype[0].field[14].datatype 3474528 +datatype[5].structtype[0].field[14].detailedtype "" datatype[5].structtype[0].field[15].name "f" datatype[5].structtype[0].field[15].datatype 3474528 +datatype[5].structtype[0].field[15].detailedtype "" datatype[5].structtype[0].field[16].name "g" datatype[5].structtype[0].field[16].datatype 2 +datatype[5].structtype[0].field[16].detailedtype "" datatype[5].structtype[0].field[17].name "anothersummaryfield" datatype[5].structtype[0].field[17].datatype 2 +datatype[5].structtype[0].field[17].detailedtype "" datatype[5].structtype[0].field[18].name "rankfeatures" datatype[5].structtype[0].field[18].datatype 2 +datatype[5].structtype[0].field[18].detailedtype "" datatype[5].structtype[0].field[19].name "summaryfeatures" datatype[5].structtype[0].field[19].datatype 2 +datatype[5].structtype[0].field[19].detailedtype "" datatype[5].structtype[0].field[20].name "snippet" datatype[5].structtype[0].field[20].datatype 2 +datatype[5].structtype[0].field[20].detailedtype "" datatype[5].structtype[0].field[21].name "snippet2" datatype[5].structtype[0].field[21].datatype 2 +datatype[5].structtype[0].field[21].detailedtype "" datatype[6].id 1858438651 datatype[6].structtype[0].name "streamingstruct.body" datatype[6].structtype[0].version 0 diff --git a/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg index fc9b0e315b8..d379d862626 100644 --- a/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 105061838 datatype[1].structtype[0].name "ns1" datatype[1].structtype[0].version 0 @@ -19,10 +21,13 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "nf1" datatype[1].structtype[0].field[0].datatype 3474528 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "nf1s" datatype[1].structtype[0].field[1].datatype 3474528 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "nf2" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[2].id 3474528 datatype[2].structtype[0].name "s1" datatype[2].structtype[0].version 0 @@ -32,12 +37,16 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "f1" datatype[2].structtype[0].field[0].datatype 2 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "f1s" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "f2" datatype[2].structtype[0].field[2].datatype 0 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "f3" datatype[2].structtype[0].field[3].datatype 5 +datatype[2].structtype[0].field[3].detailedtype "" datatype[3].id -1497802371 datatype[3].maptype[0].keytype 4 datatype[3].maptype[0].valtype 2 @@ -52,38 +61,55 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "coupleof" datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" datatype[5].structtype[0].field[1].name "normalfields" datatype[5].structtype[0].field[1].datatype 2 +datatype[5].structtype[0].field[1].detailedtype "" datatype[5].structtype[0].field[2].name "a" datatype[5].structtype[0].field[2].datatype 3474528 +datatype[5].structtype[0].field[2].detailedtype "" datatype[5].structtype[0].field[3].name "m" datatype[5].structtype[0].field[3].datatype -1497802371 +datatype[5].structtype[0].field[3].detailedtype "" datatype[5].structtype[0].field[4].name "b" datatype[5].structtype[0].field[4].datatype 3474528 +datatype[5].structtype[0].field[4].detailedtype "" datatype[5].structtype[0].field[5].name "c" datatype[5].structtype[0].field[5].datatype 3474528 +datatype[5].structtype[0].field[5].detailedtype "" datatype[5].structtype[0].field[6].name "c2" datatype[5].structtype[0].field[6].datatype 3474528 +datatype[5].structtype[0].field[6].detailedtype "" datatype[5].structtype[0].field[7].name "c3" datatype[5].structtype[0].field[7].datatype 3474528 +datatype[5].structtype[0].field[7].detailedtype "" datatype[5].structtype[0].field[8].name "n" datatype[5].structtype[0].field[8].datatype 105061838 +datatype[5].structtype[0].field[8].detailedtype "" datatype[5].structtype[0].field[9].name "array1" datatype[5].structtype[0].field[9].datatype -1425630723 +datatype[5].structtype[0].field[9].detailedtype "" datatype[5].structtype[0].field[10].name "array2" datatype[5].structtype[0].field[10].datatype -1425630723 +datatype[5].structtype[0].field[10].detailedtype "" datatype[5].structtype[0].field[11].name "array3" datatype[5].structtype[0].field[11].datatype -1425630723 +datatype[5].structtype[0].field[11].detailedtype "" datatype[5].structtype[0].field[12].name "subject" datatype[5].structtype[0].field[12].datatype 3474528 +datatype[5].structtype[0].field[12].detailedtype "" datatype[5].structtype[0].field[13].name "d" datatype[5].structtype[0].field[13].datatype 3474528 +datatype[5].structtype[0].field[13].detailedtype "" datatype[5].structtype[0].field[14].name "e" datatype[5].structtype[0].field[14].datatype 3474528 +datatype[5].structtype[0].field[14].detailedtype "" datatype[5].structtype[0].field[15].name "f" datatype[5].structtype[0].field[15].datatype 3474528 +datatype[5].structtype[0].field[15].detailedtype "" datatype[5].structtype[0].field[16].name "g" datatype[5].structtype[0].field[16].datatype 2 +datatype[5].structtype[0].field[16].detailedtype "" datatype[6].id 1858438651 datatype[6].structtype[0].name "streamingstruct.body" datatype[6].structtype[0].version 0 diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index dc56ed1cfc9..b4777f7535f 100755 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 109267174 datatype[1].structtype[0].name "sct" datatype[1].structtype[0].version 0 @@ -19,12 +21,16 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "s1" datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "s2" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "s3" datatype[1].structtype[0].field[2].datatype 109267174 +datatype[1].structtype[0].field[2].detailedtype "" datatype[1].structtype[0].field[3].name "s4" datatype[1].structtype[0].field[3].datatype 97614088 +datatype[1].structtype[0].field[3].detailedtype "" datatype[2].id 97614088 datatype[2].structtype[0].name "foo" datatype[2].structtype[0].version 0 @@ -34,6 +40,7 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "s1" datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[0].detailedtype "" datatype[3].id 517946310 datatype[3].structtype[0].name "annotation.banana" datatype[3].structtype[0].version 0 @@ -43,8 +50,10 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "b" datatype[3].structtype[0].field[0].datatype 109267174 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "c" datatype[3].structtype[0].field[1].datatype 97614088 +datatype[3].structtype[0].field[1].detailedtype "" datatype[4].id -1244829667 datatype[4].arraytype[0].datatype 109267174 datatype[5].id -364910881 @@ -56,12 +65,16 @@ datatype[5].structtype[0].compressthreshold 95 datatype[5].structtype[0].compressminsize 800 datatype[5].structtype[0].field[0].name "structfield" datatype[5].structtype[0].field[0].datatype 109267174 +datatype[5].structtype[0].field[0].detailedtype "" datatype[5].structtype[0].field[1].name "structarrayfield" datatype[5].structtype[0].field[1].datatype -1244829667 +datatype[5].structtype[0].field[1].detailedtype "" datatype[5].structtype[0].field[2].name "rankfeatures" datatype[5].structtype[0].field[2].datatype 2 +datatype[5].structtype[0].field[2].detailedtype "" datatype[5].structtype[0].field[3].name "summaryfeatures" datatype[5].structtype[0].field[3].datatype 2 +datatype[5].structtype[0].field[3].detailedtype "" datatype[6].id -1503592268 datatype[6].structtype[0].name "annotationsimplicitstruct.body" datatype[6].structtype[0].version 0 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index a3026d579d5..8819e28b354 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -16,7 +16,7 @@ attribute[0].arity 8 attribute[0].lowerbound -9223372036854775808 attribute[0].upperbound 9223372036854775807 attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" +attribute[0].tensortype "tensor(x[2],y[])" attribute[1].name "f3" attribute[1].datatype TENSOR attribute[1].collectiontype SINGLE @@ -35,7 +35,7 @@ attribute[1].arity 8 attribute[1].lowerbound -9223372036854775808 attribute[1].upperbound 9223372036854775807 attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" +attribute[1].tensortype "tensor(x{})" attribute[2].name "f4" attribute[2].datatype TENSOR attribute[2].collectiontype SINGLE diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 2a76458a97a..7c39af767f4 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -24,26 +24,32 @@ documenttype[0].datatype[0].sstruct.field[0].name "f1" documenttype[0].datatype[0].sstruct.field[0].id 26661415 documenttype[0].datatype[0].sstruct.field[0].id_v6 1740179945 documenttype[0].datatype[0].sstruct.field[0].datatype 21 +documenttype[0].datatype[0].sstruct.field[2].detailedtype "tensor(x[])" documenttype[0].datatype[0].sstruct.field[1].name "f2" documenttype[0].datatype[0].sstruct.field[1].id 2080644671 documenttype[0].datatype[0].sstruct.field[1].id_v6 1424572148 documenttype[0].datatype[0].sstruct.field[1].datatype 21 +documenttype[0].datatype[0].sstruct.field[2].detailedtype "tensor(x[2],y[])" documenttype[0].datatype[0].sstruct.field[2].name "f3" documenttype[0].datatype[0].sstruct.field[2].id 1295091863 documenttype[0].datatype[0].sstruct.field[2].id_v6 1444109654 documenttype[0].datatype[0].sstruct.field[2].datatype 21 +documenttype[0].datatype[0].sstruct.field[2].detailedtype "tensor(x{})" documenttype[0].datatype[0].sstruct.field[3].name "f4" documenttype[0].datatype[0].sstruct.field[3].id 1224191509 documenttype[0].datatype[0].sstruct.field[3].id_v6 1039544782 documenttype[0].datatype[0].sstruct.field[3].datatype 21 +documenttype[0].datatype[0].sstruct.field[3].detailedtype "tensor(x[],y[])" documenttype[0].datatype[0].sstruct.field[4].name "rankfeatures" documenttype[0].datatype[0].sstruct.field[4].id 1883197392 documenttype[0].datatype[0].sstruct.field[4].id_v6 699950698 documenttype[0].datatype[0].sstruct.field[4].datatype 2 +documenttype[0].datatype[0].sstruct.field[4].detailedtype "" documenttype[0].datatype[0].sstruct.field[5].name "summaryfeatures" documenttype[0].datatype[0].sstruct.field[5].id 1840337115 documenttype[0].datatype[0].sstruct.field[5].id_v6 1981648971 documenttype[0].datatype[0].sstruct.field[5].datatype 2 +documenttype[0].datatype[0].sstruct.field[5].detailedtype "" documenttype[0].datatype[1].id -1903234535 documenttype[0].datatype[1].type STRUCT documenttype[0].datatype[1].array.element.id 0 diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index d70555c9131..d9c6c5088c2 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -1,4 +1,8 @@ rankprofile[].name "default" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor(x[2],y[])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].name "unranked" @@ -10,6 +14,10 @@ rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor(x[2],y[])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].name "profile1" @@ -17,6 +25,10 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" rankprofile[].fef.property[].value "map(attribute(f4), f(x)(x * x)) + reduce(tensor(x[2],y[3])(random), count) * rename(attribute(f4), (x, y), (y, x))" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor(x[2],y[])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].name "profile2" @@ -24,5 +36,9 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" rankprofile[].fef.property[].value "reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x)" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor(x[2],y[])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" \ No newline at end of file diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index fab4cafddab..18f1245e0b4 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -2,16 +2,16 @@ search tensor { document tensor { - field f1 type tensor { + field f1 type tensor(x[]) { indexing: summary } - field f2 type tensor { + field f2 type tensor(x[2],y[]) { indexing: attribute } - field f3 type tensor { + field f3 type tensor(x{}) { indexing: attribute | summary } - field f4 type tensor { + field f4 type tensor(x[],y[]) { indexing: attribute attribute: tensor(x[10],y[20]) } diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index f20562d2a3c..d5679a25ed5 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -206,4 +206,4 @@ attribute[10].arity 8 attribute[10].lowerbound -9223372036854775808 attribute[10].upperbound 9223372036854775807 attribute[10].densepostinglistthreshold 0.4 -attribute[10].tensortype "" +attribute[10].tensortype "" \ No newline at end of file diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index a2018ffa195..ca9a300ad9c 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1865479609 datatype[1].maptype[0].keytype 2 datatype[1].maptype[0].valtype 4 @@ -22,12 +24,16 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "Version" datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "Name" datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[2].structtype[0].field[2].name "FlagsCounter" datatype[2].structtype[0].field[2].datatype -1865479609 +datatype[2].structtype[0].field[2].detailedtype "" datatype[2].structtype[0].field[3].name "anotherfolder" datatype[2].structtype[0].field[3].datatype 294108848 +datatype[2].structtype[0].field[3].detailedtype "" datatype[3].id 109267174 datatype[3].structtype[0].name "sct" datatype[3].structtype[0].version 0 @@ -37,8 +43,10 @@ datatype[3].structtype[0].compressthreshold 95 datatype[3].structtype[0].compressminsize 800 datatype[3].structtype[0].field[0].name "s1" datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "s2" datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" datatype[4].id 49942803 datatype[4].arraytype[0].datatype 16 datatype[5].id 339965458 @@ -53,12 +61,16 @@ datatype[6].structtype[0].compressthreshold 95 datatype[6].structtype[0].compressminsize 800 datatype[6].structtype[0].field[0].name "bytearr" datatype[6].structtype[0].field[0].datatype 49942803 +datatype[6].structtype[0].field[0].detailedtype "" datatype[6].structtype[0].field[1].name "mymap" datatype[6].structtype[0].field[1].datatype 339965458 +datatype[6].structtype[0].field[1].detailedtype "" datatype[6].structtype[0].field[2].name "title" datatype[6].structtype[0].field[2].datatype 2 +datatype[6].structtype[0].field[2].detailedtype "" datatype[6].structtype[0].field[3].name "structfield" datatype[6].structtype[0].field[3].datatype 2 +datatype[6].structtype[0].field[3].detailedtype "" datatype[7].id -1245117006 datatype[7].arraytype[0].datatype 0 datatype[8].id 1328286588 @@ -115,60 +127,88 @@ datatype[24].structtype[0].compressthreshold 95 datatype[24].structtype[0].compressminsize 800 datatype[24].structtype[0].field[0].name "abyte" datatype[24].structtype[0].field[0].datatype 16 +datatype[24].structtype[0].field[0].detailedtype "" datatype[24].structtype[0].field[1].name "along" datatype[24].structtype[0].field[1].datatype 4 +datatype[24].structtype[0].field[1].detailedtype "" datatype[24].structtype[0].field[2].name "arrayfield" datatype[24].structtype[0].field[2].datatype -1245117006 +datatype[24].structtype[0].field[2].detailedtype "" datatype[24].structtype[0].field[3].name "setfield" datatype[24].structtype[0].field[3].datatype 1328286588 +datatype[24].structtype[0].field[3].detailedtype "" datatype[24].structtype[0].field[4].name "setfield2" datatype[24].structtype[0].field[4].datatype 18 +datatype[24].structtype[0].field[4].detailedtype "" datatype[24].structtype[0].field[5].name "setfield3" datatype[24].structtype[0].field[5].datatype 2125328771 +datatype[24].structtype[0].field[5].detailedtype "" datatype[24].structtype[0].field[6].name "setfield4" datatype[24].structtype[0].field[6].datatype 2065577986 +datatype[24].structtype[0].field[6].detailedtype "" datatype[24].structtype[0].field[7].name "tagfield" datatype[24].structtype[0].field[7].datatype 18 +datatype[24].structtype[0].field[7].detailedtype "" datatype[24].structtype[0].field[8].name "structfield" datatype[24].structtype[0].field[8].datatype 109267174 +datatype[24].structtype[0].field[8].detailedtype "" datatype[24].structtype[0].field[9].name "structarrayfield" datatype[24].structtype[0].field[9].datatype -1244829667 +datatype[24].structtype[0].field[9].detailedtype "" datatype[24].structtype[0].field[10].name "stringmapfield" datatype[24].structtype[0].field[10].datatype 339965458 +datatype[24].structtype[0].field[10].detailedtype "" datatype[24].structtype[0].field[11].name "intmapfield" datatype[24].structtype[0].field[11].datatype -1584287606 +datatype[24].structtype[0].field[11].detailedtype "" datatype[24].structtype[0].field[12].name "floatmapfield" datatype[24].structtype[0].field[12].datatype 2125154557 +datatype[24].structtype[0].field[12].detailedtype "" datatype[24].structtype[0].field[13].name "longmapfield" datatype[24].structtype[0].field[13].datatype -1715531035 +datatype[24].structtype[0].field[13].detailedtype "" datatype[24].structtype[0].field[14].name "doublemapfield" datatype[24].structtype[0].field[14].datatype 2138385264 +datatype[24].structtype[0].field[14].detailedtype "" datatype[24].structtype[0].field[15].name "arraymapfield" datatype[24].structtype[0].field[15].datatype 435886609 +datatype[24].structtype[0].field[15].detailedtype "" datatype[24].structtype[0].field[16].name "arrarr" datatype[24].structtype[0].field[16].datatype -794985308 +datatype[24].structtype[0].field[16].detailedtype "" datatype[24].structtype[0].field[17].name "maparr" datatype[24].structtype[0].field[17].datatype 69621385 +datatype[24].structtype[0].field[17].detailedtype "" datatype[24].structtype[0].field[18].name "mystructfield" datatype[24].structtype[0].field[18].datatype -2092985853 +datatype[24].structtype[0].field[18].detailedtype "" datatype[24].structtype[0].field[19].name "mystructmap" datatype[24].structtype[0].field[19].datatype 1901258752 +datatype[24].structtype[0].field[19].detailedtype "" datatype[24].structtype[0].field[20].name "mystructarr" datatype[24].structtype[0].field[20].datatype 759956026 +datatype[24].structtype[0].field[20].detailedtype "" datatype[24].structtype[0].field[21].name "Folders" datatype[24].structtype[0].field[21].datatype -389833101 +datatype[24].structtype[0].field[21].detailedtype "" datatype[24].structtype[0].field[22].name "juletre" datatype[24].structtype[0].field[22].datatype 4 +datatype[24].structtype[0].field[22].detailedtype "" datatype[24].structtype[0].field[23].name "album0" datatype[24].structtype[0].field[23].datatype 18 +datatype[24].structtype[0].field[23].detailedtype "" datatype[24].structtype[0].field[24].name "album1" datatype[24].structtype[0].field[24].datatype 18 +datatype[24].structtype[0].field[24].detailedtype "" datatype[24].structtype[0].field[25].name "other" datatype[24].structtype[0].field[25].datatype 4 +datatype[24].structtype[0].field[25].detailedtype "" datatype[24].structtype[0].field[26].name "rankfeatures" datatype[24].structtype[0].field[26].datatype 2 +datatype[24].structtype[0].field[26].detailedtype "" datatype[24].structtype[0].field[27].name "summaryfeatures" datatype[24].structtype[0].field[27].datatype 2 +datatype[24].structtype[0].field[27].detailedtype "" datatype[25].id 171503364 datatype[25].maptype[0].keytype 1707615575 datatype[25].maptype[0].valtype 0 @@ -183,6 +223,7 @@ datatype[27].structtype[0].compressthreshold 95 datatype[27].structtype[0].compressminsize 800 datatype[27].structtype[0].field[0].name "complexarray" datatype[27].structtype[0].field[0].datatype 1100964733 +datatype[27].structtype[0].field[0].detailedtype "" datatype[28].id -853072901 datatype[28].documenttype[0].name "types" datatype[28].documenttype[0].version 0 diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index cc692e120be..70b799d0df0 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -52,4 +52,4 @@ ilscript[0].content[22] "input mystructfield | passthrough mystructfield" ilscript[0].content[23] "input mystructmap | passthrough mystructmap" ilscript[0].content[24] "input stringmapfield | passthrough stringmapfield" ilscript[0].content[25] "input structarrayfield | passthrough structarrayfield" -ilscript[0].content[26] "input structfield | passthrough structfield" +ilscript[0].content[26] "input structfield | passthrough structfield" \ No newline at end of file diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 4b94127c687..85d4c884571 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -424,4 +424,4 @@ indexinfo[0].command[210].command "multivalue" indexinfo[0].command[211].indexname "rankfeatures" indexinfo[0].command[211].command "index" indexinfo[0].command[212].indexname "summaryfeatures" -indexinfo[0].command[212].command "index" +indexinfo[0].command[212].command "index" \ No newline at end of file diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index 6a62dedb851..b1205f45709 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -32,4 +32,4 @@ classes[1].fields[3].type "int64" classes[1].fields[4].name "rankfeatures" classes[1].fields[4].type "featuredata" classes[1].fields[5].name "summaryfeatures" -classes[1].fields[5].type "featuredata" +classes[1].fields[5].type "featuredata" \ No newline at end of file diff --git a/config-model/src/test/derived/types/summarymap.cfg b/config-model/src/test/derived/types/summarymap.cfg index eac532d966b..8a3ed769213 100644 --- a/config-model/src/test/derived/types/summarymap.cfg +++ b/config-model/src/test/derived/types/summarymap.cfg @@ -22,4 +22,4 @@ override[6].command "attribute" override[6].arguments "other" override[7].field "juletre" override[7].command "attribute" -override[7].arguments "juletre" +override[7].arguments "juletre" \ No newline at end of file diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index 9ce928381ec..d3dd0d06b66 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id -1344444812 datatype[1].structtype[0].name "book.header" datatype[1].structtype[0].version 0 @@ -19,10 +21,13 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "soundtrack" datatype[1].structtype[0].field[0].datatype 1412693671 +datatype[1].structtype[0].field[0].detailedtype "" datatype[1].structtype[0].field[1].name "rankfeatures" datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].detailedtype "" datatype[1].structtype[0].field[2].name "summaryfeatures" datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[2].detailedtype "" datatype[2].id -820813431 datatype[2].structtype[0].name "book.body" datatype[2].structtype[0].version 0 @@ -47,14 +52,19 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "intfield" datatype[4].structtype[0].field[0].datatype 0 +datatype[4].structtype[0].field[0].detailedtype "" datatype[4].structtype[0].field[1].name "stringfield" datatype[4].structtype[0].field[1].datatype 2 +datatype[4].structtype[0].field[1].detailedtype "" datatype[4].structtype[0].field[2].name "longfield" datatype[4].structtype[0].field[2].datatype 4 +datatype[4].structtype[0].field[2].detailedtype "" datatype[4].structtype[0].field[3].name "rankfeatures" datatype[4].structtype[0].field[3].datatype 2 +datatype[4].structtype[0].field[3].detailedtype "" datatype[4].structtype[0].field[4].name "summaryfeatures" datatype[4].structtype[0].field[4].datatype 2 +datatype[4].structtype[0].field[4].detailedtype "" datatype[5].id 993120973 datatype[5].structtype[0].name "music.body" datatype[5].structtype[0].version 0 diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index addb5f6205c..075ac54b983 100755 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -8,8 +8,10 @@ datatype[0].structtype[0].compressthreshold 95 datatype[0].structtype[0].compressminsize 800 datatype[0].structtype[0].field[0].name "x" datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" datatype[0].structtype[0].field[1].name "y" datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" datatype[1].id 93505813 datatype[1].structtype[0].name "bar" datatype[1].structtype[0].version 0 @@ -19,6 +21,7 @@ datatype[1].structtype[0].compressthreshold 95 datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "humbe" datatype[1].structtype[0].field[0].datatype 97614088 +datatype[1].structtype[0].field[0].detailedtype "" datatype[2].id 97614088 datatype[2].structtype[0].name "foo" datatype[2].structtype[0].version 0 @@ -28,9 +31,11 @@ datatype[2].structtype[0].compressthreshold 95 datatype[2].structtype[0].compressminsize 800 datatype[2].structtype[0].field[0].name "fubar" datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "bar" datatype[2].structtype[0].field[1].id[0].id 1 datatype[2].structtype[0].field[1].datatype 2 +datatype[2].structtype[0].field[1].detailedtype "" datatype[3].id -1245205573 datatype[3].arraytype[0].datatype 97614088 datatype[4].id -1910204744 @@ -42,14 +47,19 @@ datatype[4].structtype[0].compressthreshold 95 datatype[4].structtype[0].compressminsize 800 datatype[4].structtype[0].field[0].name "mystruct" datatype[4].structtype[0].field[0].datatype 97614088 +datatype[4].structtype[0].field[0].detailedtype "" datatype[4].structtype[0].field[1].name "arraystruct" datatype[4].structtype[0].field[1].datatype -1245205573 +datatype[4].structtype[0].field[1].detailedtype "" datatype[4].structtype[0].field[2].name "advanced" datatype[4].structtype[0].field[2].datatype 93505813 +datatype[4].structtype[0].field[2].detailedtype "" datatype[4].structtype[0].field[3].name "rankfeatures" datatype[4].structtype[0].field[3].datatype 2 +datatype[4].structtype[0].field[3].detailedtype "" datatype[4].structtype[0].field[4].name "summaryfeatures" datatype[4].structtype[0].field[4].datatype 2 +datatype[4].structtype[0].field[4].detailedtype "" datatype[5].id 993120973 datatype[5].structtype[0].name "music.body" datatype[5].structtype[0].version 0 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java index 4a3119e55b7..83d5c422fdf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java @@ -14,9 +14,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase { + @Test public void testDocument() throws IOException, ParseException { @@ -25,7 +26,7 @@ public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase { sds.add("src/test/examples/fieldoftypedocument.sd"); DocumentmanagerConfig.Builder value = Deriver.getDocumentManagerConfig(sds); assertConfigFile("src/test/examples/fieldoftypedocument.cfg", - new DocumentmanagerConfig(value).toString() + "\n"); + new DocumentmanagerConfig(value).toString() + "\n"); DocumentTypeManager manager = new DocumentTypeManager(); DocumentTypeManagerConfigurer.configure(manager, "raw:" + new DocumentmanagerConfig(value).toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index f8fe979c866..5542c1ce5e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -6,14 +6,16 @@ import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + /** * Tests that "name" is not allowed as name for a field. * * And that duplicate names are not allowed. * - * @author Lars Christian Jensen + * @author Lars Christian Jensen */ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { @@ -22,8 +24,8 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { try { SearchBuilder.buildFromFile("src/test/examples/name-check.sd"); fail("Should throw exception."); - } catch (Exception e) { - e.printStackTrace(); + } catch (Exception expected) { + // Success } } @@ -33,8 +35,7 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { SearchBuilder.buildFromFile("src/test/examples/duplicatenamesinsearchdifferenttype.sd"); fail("Should throw exception."); } catch (Exception e) { - e.printStackTrace(); - assertTrue(e.getMessage().matches(".*Duplicate.*different type.*")); + assertEquals("For search 'duplicatenamesinsearch', field 'grpphotoids64': Incompatible types. Expected Array for index field 'grpphotoids64', got string.", e.getMessage()); } } @@ -44,7 +45,6 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { SearchBuilder.buildFromFile("src/test/examples/duplicatenamesindoc.sd"); fail("Should throw exception."); } catch (Exception e) { - e.printStackTrace(); assertTrue(e.getMessage().matches(".*Duplicate.*")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index ae3b775f13d..b2968eb4a85 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -102,9 +102,9 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { SearchBuilder builder = new SearchBuilder(registry); builder.importString("search test {\n" + " document test { \n" + - " field a type tensor { indexing: attribute \n attribute: tensor(x[10]) }\n" + - " field b type tensor { indexing: attribute \n attribute: tensor(y{}) }\n" + - " field c type tensor { indexing: attribute }\n" + + " field a type tensor(x[]) { indexing: attribute \n attribute: tensor(x[10]) }\n" + + " field b type tensor(y{}) { indexing: attribute \n attribute: tensor(y{}) }\n" + + " field c type tensor(x[]) { indexing: attribute }\n" + " }\n" + " rank-profile p1 {}\n" + " rank-profile p2 {}\n" + @@ -123,7 +123,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); - assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").isPresent()); + assertEquals("tensor(x[])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index c33ebc95bd2..3236be983d8 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -10,26 +10,28 @@ import java.io.IOException; import static org.junit.Assert.fail; /** - * tests importing of document containing array type fields + * Tests importing of document containing array type fields * * @author bratseth */ public class StructTestCase extends SearchDefinitionTestCase { + @Test public void testStruct() throws IOException, ParseException { assertConfigFile("src/test/examples/structresult.cfg", - new DocumentmanagerConfig(Deriver.getDocumentManagerConfig("src/test/examples/struct.sd")).toString() + "\n"); + new DocumentmanagerConfig(Deriver.getDocumentManagerConfig("src/test/examples/struct.sd")).toString() + "\n"); } + @Test public void testBadStruct() throws IOException { try { SearchBuilder.buildFromFile("src/test/examples/badstruct.sd"); fail("Should throw exception."); - } catch (ParseException e) { - //ok! - //e.printStackTrace(); + } catch (ParseException expected) { + // success } } + @Test public void testStructAndDocumentWithSameNames() throws IOException, ParseException { try { @@ -41,8 +43,6 @@ public class StructTestCase extends SearchDefinitionTestCase { /** * Declaring a struct before a document will fail, no doc type to add it to. - * @throws IOException - * @throws ParseException */ @Test(expected = IllegalArgumentException.class) public void testStructOutsideDocumentIllegal() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index ecadf8e1b71..0007019e6b4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -19,28 +19,28 @@ public class TensorFieldTestCase { public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { exception.expect(IllegalArgumentException.class); exception.expectMessage("For search 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); - SearchBuilder.createFromString(getSd("field f1 type array {}")); + SearchBuilder.createFromString(getSd("field f1 type array {}")); } @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { exception.expect(IllegalArgumentException.class); exception.expectMessage("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field."); - SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: index }")); + SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); } @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { exception.expect(IllegalArgumentException.class); exception.expectMessage("For search 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'."); - SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: attribute \n attribute: fast-search }")); + SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); } @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { exception.expect(IllegalArgumentException.class); exception.expectMessage("For attribute field 'f1': Illegal tensor type spec: Failed parsing element 'invalid' in type spec 'tensor(invalid)'"); - SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: attribute \n attribute: tensor(invalid) }")); + SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: tensor(invalid) }")); } private static String getSd(String field) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java index 8c1a288c46d..a8b4065ef38 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java @@ -8,7 +8,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * @author Simon Thoresen + * @author Simon Thoresen */ public class SearchDataTypeValidatorTestCase { @@ -37,4 +37,5 @@ public class SearchDataTypeValidatorTestCase { "'simple' for cluster 'content').", e.getMessage()); } } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index b3ef8c6d7f2..e43827d7693 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -9,6 +9,7 @@ import java.util.List; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; +import static org.junit.Assert.assertEquals; public class AttributeChangeValidatorTest { @@ -110,8 +111,8 @@ public class AttributeChangeValidatorTest { @Test public void requireThatChangingTensorTypeOfTensorFieldRequiresRefeed() throws Exception { new Fixture( - "field f1 type tensor { indexing: attribute \n attribute: tensor(x[100]) }", - "field f1 type tensor { indexing: attribute \n attribute: tensor(y[]) }") + "field f1 type tensor(x[]) { indexing: attribute \n attribute: tensor(x[100]) }", + "field f1 type tensor(y[]) { indexing: attribute \n attribute: tensor(y[]) }") .assertValidation(newRefeedAction( "tensor-type-change", ValidationOverrides.empty(), @@ -119,10 +120,24 @@ public class AttributeChangeValidatorTest { } @Test - public void requireThatNotChangingTensorTypeOfTensorFieldIsOk() throws Exception { + public void requireThatCompatibleTensorTypeChangeIsOk() throws Exception { new Fixture( - "field f1 type tensor { indexing: attribute \n attribute: tensor(x[104], y[52]) }", - "field f1 type tensor { indexing: attribute \n attribute: tensor(x[104], y[52]) }") + "field f1 type tensor(x[],y[]) { indexing: attribute \n attribute: tensor(x[104], y[52]) }", + "field f1 type tensor(x[200],y[]) { indexing: attribute \n attribute: tensor(x[104], y[52]) }") .assertValidation(); } + + @Test + public void requireIncompatibleTensorTypeChangeIsNotOk() throws Exception { + try { + new Fixture( + "field f1 type tensor(x[],y[]) { indexing: attribute \n attribute: tensor(x[104], y[52]) }", + "field f1 type tensor(x[100],y[]) { indexing: attribute \n attribute: tensor(x[104], y[52]) }") + .assertValidation(); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'f1': Incompatible types. Expected tensor(x[100],y[]) for attribute 'f1', got tensor(x[104],y[52]).", e.getMessage()); + } + } + } diff --git a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java index 8c209438565..5cf9aeda978 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.search.query.ranking.RankProperties; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import org.junit.Test; @@ -30,16 +31,18 @@ public class RankFeaturesTestCase { @Test public void requireThatSingleTensorIsBinaryEncoded() { - Tensor tensor = Tensor.from("{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); - assertTensorEncodingAndDecoding("query(my_tensor)", "my_tensor", tensor); - assertTensorEncodingAndDecoding("$my_tensor", "my_tensor", tensor); + TensorType type = new TensorType.Builder().mapped("x").mapped("y").mapped("z").build(); + Tensor tensor = Tensor.from(type, "{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); + assertTensorEncodingAndDecoding(type, "query(my_tensor)", "my_tensor", tensor); + assertTensorEncodingAndDecoding(type, "$my_tensor", "my_tensor", tensor); } @Test public void requireThatMultipleTensorsAreBinaryEncoded() { - Tensor tensor1 = Tensor.from("{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); - Tensor tensor2 = Tensor.from("{ {x:a, y:b, z:c}:5.0 }"); - assertTensorEncodingAndDecoding(Arrays.asList( + TensorType type = new TensorType.Builder().mapped("x").mapped("y").mapped("z").build(); + Tensor tensor1 = Tensor.from(type, "{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); + Tensor tensor2 = Tensor.from(type, "{ {x:a, y:b, z:c}:5.0 }"); + assertTensorEncodingAndDecoding(type, Arrays.asList( new Entry("query(tensor1)", "tensor1", tensor1), new Entry("$tensor2", "tensor2", tensor2))); } @@ -55,11 +58,11 @@ public class RankFeaturesTestCase { } } - private static void assertTensorEncodingAndDecoding(List entries) { + private static void assertTensorEncodingAndDecoding(TensorType type, List entries) { RankProperties properties = createRankPropertiesWithTensors(entries); assertEquals(entries.size(), properties.asMap().size()); - Map decodedProperties = decode(encode(properties)); + Map decodedProperties = decode(type, encode(properties)); assertEquals(entries.size() * 2, properties.asMap().size()); // tensor type info has been added assertEquals(entries.size() * 2, decodedProperties.size()); for (Entry entry : entries) { @@ -68,8 +71,8 @@ public class RankFeaturesTestCase { } } - private static void assertTensorEncodingAndDecoding(String key, String normalizedKey, Tensor tensor) { - assertTensorEncodingAndDecoding(Arrays.asList(new Entry(key, normalizedKey, tensor))); + private static void assertTensorEncodingAndDecoding(TensorType type, String key, String normalizedKey, Tensor tensor) { + assertTensorEncodingAndDecoding(type, Arrays.asList(new Entry(key, normalizedKey, tensor))); } private static RankProperties createRankPropertiesWithTensors(List entries) { @@ -91,7 +94,7 @@ public class RankFeaturesTestCase { return result; } - private static Map decode(byte[] encodedProperties) { + private static Map decode(TensorType type, byte[] encodedProperties) { GrowableByteBuffer buffer = GrowableByteBuffer.wrap(encodedProperties); byte[] mapNameBytes = new byte[buffer.getInt()]; buffer.get(mapNameBytes); @@ -106,7 +109,7 @@ public class RankFeaturesTestCase { if (key.contains(".type")) { result.put(key, Utf8.toString(value)); } else { - result.put(key, TypedBinaryFormat.decode(null, value)); // TODO: Pass type + result.put(key, TypedBinaryFormat.decode(type, value)); } } return result; diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java index 640bd94bd1c..4b24bb1ae00 100644 --- a/document/src/main/java/com/yahoo/document/ArrayDataType.java +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -8,9 +8,10 @@ import java.util.ArrayList; import java.util.List; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class ArrayDataType extends CollectionDataType { + // The global class identifier shared with C++. public static int classId = registerClass(Ids.document + 54, ArrayDataType.class); diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 51af799efd9..184340f295e 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -14,6 +14,7 @@ import com.yahoo.document.datatypes.Raw; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.UriFieldValue; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Ids; import com.yahoo.vespa.objects.ObjectVisitor; @@ -46,40 +47,30 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory()); public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory()); public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory()); - // ARRAY is type 6, but never used, array IDs are generated - // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class); public final static DocumentType DOCUMENT = new DocumentType("document"); - // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); - // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class); - // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); - // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated - // Tags are converted to weightedset<string> when reading the search definition - public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); - // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class); - // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now.. - // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); - public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory()); + public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately + // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor + + // Tags are converted to weightedset when reading the search definition TODO: Remove it + public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); public static int lastPredefinedDataTypeId() { return 21; } - /** - * Set to true when this type is registered in a type manager. From that time we should refuse changes. - */ + /** Set to true when this type is registered in a type manager. From that time we should refuse changes. */ private boolean registered = false; private String name; - /** - * The id of this type - */ + /** The id of this type */ private int dataTypeId; static final private CopyOnWriteHashMap constructorCache = new CopyOnWriteHashMap<>(); + /** * Creates a datatype * @@ -96,26 +87,21 @@ public abstract class DataType extends Identifiable implements Serializable, Com return (DataType)super.clone(); } - public void setRegistered() { + void setRegistered() { registered = true; } - public boolean isRegistered() { return registered; } - /** - * Creates a new, empty FieldValue of this type. - * - * @return a new, empty FieldValue of this type. - */ + /** Creates a new, empty FieldValue of this type */ public abstract FieldValue createFieldValue(); /** - * This will try to create the object by reflection. This can be very expensive - * so some might discourage that. - * @param arg The constructor argument. - * @return Fully constructed value. + * Creates a field value by reflection + * + * @param arg the value of the newly created field value + * @return a fully constructed value */ protected FieldValue createByReflection(Object arg) { Class valClass = getValueClass(); @@ -141,8 +127,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com /** * Creates a new FieldValue of this type, with the given value. * - * @param arg the value that the new FieldValue shall have. - * @return A new FieldValue of this type, with the given value. + * @param arg the value of the new FieldValue + * @return a new FieldValue of this type, with the given value */ public FieldValue createFieldValue(Object arg) { if (arg == null) { @@ -211,12 +197,17 @@ public abstract class DataType extends Identifiable implements Serializable, Com return new WeightedSetDataType(type, createIfNonExistent, removeIfZero); } + /** Returns the given tensor type as a DataType */ + public static TensorDataType getTensor(TensorType type) { + return new TensorDataType(type); + } + public String getName() { return name; } /** - * Sets the name of this data type. WARNING! Do not use! + * Sets the name of this data type. WARNING! Do not use! * * @param name the name of this data type. */ @@ -229,7 +220,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Sets the ID of this data type. WARNING! Do not use! + * Sets the ID of this data type. WARNING! Do not use! * * @param id the ID of this data type. */ @@ -238,7 +229,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com } /** - * Registeres this type in the given document manager. + * Registers this type in the given document manager. * * @param manager the DocumentTypeManager to register in. */ @@ -255,14 +246,12 @@ public abstract class DataType extends Identifiable implements Serializable, Com } public boolean equals(Object other) { - if (!(other instanceof DataType)) { - return false; - } + if (!(other instanceof DataType)) return false; DataType type = (DataType)other; return (name.equals(type.name) && dataTypeId == type.dataTypeId); } - public java.lang.String toString() { + public String toString() { return "datatype " + name + " (code: " + dataTypeId + ")"; } @@ -278,8 +267,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com */ public FieldPath buildFieldPath(String fieldPathString) { if (fieldPathString.length() > 0) { - throw new IllegalArgumentException( - "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString); + throw new IllegalArgumentException("Datatype " + toString() + + " does not support further recursive structure: " + fieldPathString); } return new FieldPath(); } diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 34c952e1cec..e1d912b4e51 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -26,7 +26,7 @@ import java.util.Map; * be removed soon. * * @author bratseth - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class Document extends StructuredFieldValue { @@ -267,7 +267,9 @@ public class Document extends StructuredFieldValue { } /** Returns true if the argument is a document which has the same set of values */ + @Override public boolean equals(Object o) { + if (o == this) return true; if (!(o instanceof Document)) return false; Document other = (Document) o; return (super.equals(o) && docId.equals(other.docId) && @@ -394,4 +396,5 @@ public class Document extends StructuredFieldValue { comp = body.compareTo(otherValue.body); return comp; } + } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index 0de7fe60500..1b3370cfa66 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -12,6 +12,7 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.VespaDocumentDeserializer42; import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.TensorType; import java.lang.reflect.Modifier; import java.util.*; @@ -28,13 +29,23 @@ import java.util.logging.Logger; * datatypes, for instance displaying the data type in human-readable form * or as XML. * - * @author Thomas Gundersen + * @author Thomas Gundersen */ public class DocumentTypeManager { private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); private ConfigSubscriber subscriber; + // *Configured data types* (not built-in/primitive) indexed by their id + // + // *Primitive* data types are always available and have a single id. + // + // *Built-in dynamic* types: The tensor type. + // Any tensor type has the same id and is always available just like primitive types. + // However, unlike primitive types, each tensor type is a separate DataType instance + // (which carries additional type information (detailedType) supplied at runtime). + // The reason for this is that we want the data type to stay the same when we change the detailed tensor type + // to maintain compatibility on (some) tensor type changes. private Map dataTypes = new LinkedHashMap<>(); private Map documentTypes = new LinkedHashMap<>(); private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry(); @@ -85,6 +96,7 @@ public class DocumentTypeManager { } public boolean hasDataType(String name) { + if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { return true; @@ -94,10 +106,14 @@ public class DocumentTypeManager { } public boolean hasDataType(int code) { + if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present return dataTypes.containsKey(code); } public DataType getDataType(String name) { + if (name.startsWith("tensor(")) // built-in dynamic + return new TensorDataType(TensorType.fromSpec(name)); + List foundTypes = new ArrayList<>(); for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { @@ -125,7 +141,19 @@ public class DocumentTypeManager { return foundTypes.get(0); } - public DataType getDataType(int code) { + public DataType getDataType(int code) { return getDataType(code, ""); } + + /** + * Return a data type instance + * + * @param code the code of the data type to return, which must be either built in or present in this manager + * @param detailedType detailed type information, or the empty string if none + * @return the appropriate DataType instance + */ + public DataType getDataType(int code, String detailedType) { + if (code == DataType.tensorDataTypeCode) // built-in dynamic + return new TensorDataType(TensorType.fromSpec(detailedType)); + DataType type = dataTypes.get(code); if (type == null) { StringBuilder types=new StringBuilder(); @@ -138,11 +166,11 @@ public class DocumentTypeManager { } } - DataType getDataTypeAndReturnTemporary(int code) { + DataType getDataTypeAndReturnTemporary(int code, String detailedType) { if (hasDataType(code)) { - return getDataType(code); + return getDataType(code, detailedType); } - return new TemporaryDataType(code); + return new TemporaryDataType(code, detailedType); } /** @@ -156,9 +184,11 @@ public class DocumentTypeManager { /** * Register a single datatype. Re-registering an existing, but equal, datatype is ok. + * * @param type The datatype to register */ void registerSingleType(DataType type) { + if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly if (dataTypes.containsKey(type.getId())) { DataType existingType = dataTypes.get(type.getId()); if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType)) @@ -188,7 +218,7 @@ public class DocumentTypeManager { dataTypes.remove(existingType.getId()); } else { throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered " - + type + ", but already uses id " + type.getId()); + + type + ", but already uses id " + type.getId()); } } @@ -250,7 +280,8 @@ public class DocumentTypeManager { } /** - * A read only view of the registered data types + * Returns a read only view of the registered data types + * * @return collection of types */ public Collection getDataTypes() { @@ -318,7 +349,7 @@ public class DocumentTypeManager { for (Field field : structDataType.getFieldsThisTypeOnly()) { DataType fieldType = field.getDataType(); if (fieldType instanceof TemporaryDataType) { - field.setDataType(getDataType(fieldType.getCode())); + field.setDataType(getDataType(fieldType.getCode(), ((TemporaryDataType)fieldType).getDetailedType())); } else { if (!seenStructs.contains(fieldType)) { replaceTemporaryTypes(fieldType, seenStructs); @@ -329,7 +360,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List seenStructs) { if (collectionDataType.getNestedType() instanceof TemporaryDataType) { - collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode())); + collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode(), "")); } else { replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs); } @@ -337,13 +368,13 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInMap(MapDataType mapDataType, List seenStructs) { if (mapDataType.getValueType() instanceof TemporaryDataType) { - mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode())); + mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode(), "")); } else { replaceTemporaryTypes(mapDataType.getValueType(), seenStructs); } if (mapDataType.getKeyType() instanceof TemporaryDataType) { - mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode())); + mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode(), "")); } else { replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs); } @@ -351,7 +382,7 @@ public class DocumentTypeManager { private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List seenStructs) { if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) { - weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode())); + weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode(), "")); } else { replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs); } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 9d7139e6226..7b5f1392032 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; /** * Configures the Vespa document manager from a config id. * - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber{ @@ -82,21 +82,21 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub try { for (Object o : thisDataType.arraytype()) { DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o; - DataType nestedType = manager.getDataType(array.datatype()); + DataType nestedType = manager.getDataType(array.datatype(), ""); ArrayDataType type = new ArrayDataType(nestedType, id); manager.register(type); } for (Object o : thisDataType.maptype()) { DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o; - DataType keyType = manager.getDataType(map.keytype()); - DataType valType = manager.getDataType(map.valtype()); + DataType keyType = manager.getDataType(map.keytype(), ""); + DataType valType = manager.getDataType(map.valtype(), ""); MapDataType type = new MapDataType(keyType, valType, id); manager.register(type); } for (Object o : thisDataType.weightedsettype()) { DocumentmanagerConfig.Datatype.Weightedsettype wset = (DocumentmanagerConfig.Datatype.Weightedsettype) o; - DataType nestedType = manager.getDataType(wset.datatype()); + DataType nestedType = manager.getDataType(wset.datatype(), ""); WeightedSetDataType type = new WeightedSetDataType( nestedType, wset.createifnonexistant(), wset.removeifzero(), id); manager.register(type); @@ -114,8 +114,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentmanagerConfig.Datatype.Structtype.Field field = (DocumentmanagerConfig.Datatype.Structtype.Field) j; DataType fieldType = (field.datatype() == id) - ? manager.getDataTypeAndReturnTemporary(field.datatype()) - : manager.getDataType(field.datatype()); + ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype()) + : manager.getDataType(field.datatype(), field.detailedtype()); if (field.id().size() == 1) { type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); @@ -127,8 +127,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } for (Object o : thisDataType.documenttype()) { DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o; - StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct()); - StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct()); + StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); + StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), ""); for (Field field : body.getFields()) { field.setHeader(false); } @@ -210,7 +210,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) { for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id()); - DataType payload = manager.getDataType(annType.datatype()); + DataType payload = manager.getDataType(annType.datatype(), ""); if (!payload.equals(DataType.NONE)) { annotationType.setDataType(payload); } @@ -234,7 +234,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub int id = thisDataType.id(); for (Object o : thisDataType.structtype()) { DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; - StructDataType thisStruct = (StructDataType) manager.getDataType(id); + StructDataType thisStruct = (StructDataType) manager.getDataType(id, ""); for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 86543916b42..b80d92e0e4d 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -13,7 +13,7 @@ import java.io.Serializable; * A name and type. Fields are contained in document types to describe their fields, * but is also used to represent name/type pairs which are not part of document types. * - * @author Thomas Gundersen + * @author Thomas Gundersen * @author bratseth */ public class Field extends FieldBase implements FieldSet, Comparable, Serializable { diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java index a0024bb0497..23bf4b43ccf 100644 --- a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java +++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java @@ -8,9 +8,10 @@ import com.yahoo.vespa.objects.ObjectVisitor; import java.util.Objects; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public class PrimitiveDataType extends DataType { + public static abstract class Factory { public abstract FieldValue create(); } diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java index da65dde72da..a0c6fcb889d 100644 --- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java +++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java @@ -4,11 +4,15 @@ package com.yahoo.document; import com.yahoo.document.datatypes.FieldValue; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ class TemporaryDataType extends DataType { - TemporaryDataType(int dataTypeId) { + + private final String detailedType; + + TemporaryDataType(int dataTypeId, String detailedType) { super("temporary_" + dataTypeId, dataTypeId); + this.detailedType = detailedType; } @Override @@ -25,4 +29,7 @@ class TemporaryDataType extends DataType { public boolean isValueCompatible(FieldValue value) { return false; } + + String getDetailedType() { return detailedType; } + } diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java new file mode 100644 index 00000000000..dbaf6ee7763 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/TensorDataType.java @@ -0,0 +1,50 @@ +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.objects.Ids; + +/** + * A DataType containing a tensor type + * + * @author bratseth + */ +public class TensorDataType extends DataType { + + private final TensorType tensorType; + + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 59, TensorDataType.class); + + public TensorDataType(TensorType tensorType) { + super(tensorType.toString(), DataType.tensorDataTypeCode); + this.tensorType = tensorType; + } + + public TensorDataType clone() { + return (TensorDataType)super.clone(); + } + + @Override + public FieldValue createFieldValue() { + return new TensorFieldValue(tensorType); + } + + @Override + public Class getValueClass() { + return TensorFieldValue.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (value == null) return false; + if ( ! TensorFieldValue.class.isAssignableFrom(value.getClass())) return false; + TensorFieldValue tensorValue = (TensorFieldValue)value; + return tensorValue.getDataType().getTensorType().isAssignableTo(tensorType); + } + + /** Returns the type of the tensor this field can hold */ + public TensorType getTensorType() { return tensorType; } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 66cc472de69..09fb8b71db1 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -16,7 +16,7 @@ import java.util.*; /** * FieldValue which encapsulates a Array value * - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge */ public final class Array extends CollectionFieldValue implements List { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index 5a01dc33aa1..b5920cb4758 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -215,7 +215,6 @@ public class Struct extends StructuredFieldValue { if (!super.equals(o)) return false; Struct struct = (Struct) o; - return values.equals(struct.values); } @@ -388,4 +387,5 @@ public class Struct extends StructuredFieldValue { } return fieldType.cast(fieldValue); } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java index b4585a2188d..7957d9b812f 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -10,7 +10,7 @@ import java.util.List; import java.util.Map; /** - * @author Håkon Humberset + * @author Håkon Humberset */ public abstract class StructuredFieldValue extends CompositeFieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index bee29478219..9d8e9a83b5e 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -3,11 +3,12 @@ package com.yahoo.document.datatypes; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.serialization.FieldReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.document.serialization.XmlStream; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import java.util.Objects; import java.util.Optional; @@ -20,12 +21,18 @@ import java.util.Optional; public class TensorFieldValue extends FieldValue { private Optional tensor; + + private final TensorDataType dataType; - public TensorFieldValue() { - tensor = Optional.empty(); + /** Create an empty tensor field value */ + public TensorFieldValue(TensorType type) { + this.dataType = new TensorDataType(type); + this.tensor = Optional.empty(); } + /** Create a tensor field value containing the given tensor */ public TensorFieldValue(Tensor tensor) { + this.dataType = new TensorDataType(tensor.type()); this.tensor = Optional.of(tensor); } @@ -34,8 +41,8 @@ public class TensorFieldValue extends FieldValue { } @Override - public DataType getDataType() { - return DataType.TENSOR; + public TensorDataType getDataType() { + return dataType; } @Override @@ -51,16 +58,23 @@ public class TensorFieldValue extends FieldValue { @Override public void assign(Object o) { if (o == null) { - tensor = Optional.empty(); + assignTensor(Optional.empty()); } else if (o instanceof Tensor) { - tensor = Optional.of((Tensor)o); + assignTensor(Optional.of((Tensor)o)); } else if (o instanceof TensorFieldValue) { - tensor = ((TensorFieldValue)o).getTensor(); + assignTensor(((TensorFieldValue)o).getTensor()); } else { throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" + - o.getClass().getName() + "'."); + o.getClass().getName() + "'."); } } + + public void assignTensor(Optional tensor) { + if (tensor.isPresent() && ! dataType.getTensorType().isAssignableTo(tensor.get().type())) + throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() + + " to field of type " + dataType.getTensorType()); + this.tensor = tensor; + } @Override public void serialize(Field field, FieldWriter writer) { @@ -74,27 +88,14 @@ public class TensorFieldValue extends FieldValue { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TensorFieldValue)) { - return false; - } - TensorFieldValue rhs = (TensorFieldValue)o; - if (!Objects.equals(tensor, rhs.tensor)) { - return false; - } + if (this == o) return true; + if ( ! (o instanceof TensorFieldValue)) return false; + + TensorFieldValue other = (TensorFieldValue)o; + if ( ! dataType.getTensorType().equals(other.dataType.getTensorType())) return false; + if ( ! tensor.equals(other.tensor)) return false; return true; } - public static PrimitiveDataType.Factory getFactory() { - return new PrimitiveDataType.Factory() { - - @Override - public FieldValue create() { - return new TensorFieldValue(); - } - }; - } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index b1c635d7641..e0740338aae 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -583,71 +583,41 @@ public class JsonReader { } private void fillTensor(TensorFieldValue tensorFieldValue) { + Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); - Tensor.Builder tensorBuilder = null; // read tensor cell fields and ignore everything else for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { if (TENSOR_CELLS.equals(buffer.currentName())) - tensorBuilder = readTensorCells(tensorBuilder); + readTensorCells(tensorBuilder); } expectObjectEnd(buffer.currentToken()); - if (tensorBuilder == null) // no cells + no type: empty tensor type - tensorBuilder = Tensor.Builder.of(TensorType.empty); tensorFieldValue.assign(tensorBuilder.build()); } - private Tensor.Builder readTensorCells(Tensor.Builder tensorBuilder) { + private void readTensorCells(Tensor.Builder tensorBuilder) { expectArrayStart(buffer.currentToken()); int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - tensorBuilder = readTensorCell(tensorBuilder); - } + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + readTensorCell(tensorBuilder); expectCompositeEnd(buffer.currentToken()); - return tensorBuilder; } - private Tensor.Builder readTensorCell(Tensor.Builder tensorBuilder) { + private void readTensorCell(Tensor.Builder tensorBuilder) { expectObjectStart(buffer.currentToken()); int initNesting = buffer.nesting(); double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = null; + Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { String currentName = buffer.currentName(); if (TENSOR_ADDRESS.equals(currentName)) { - if (tensorBuilder != null) { - cellBuilder = tensorBuilder.cell(); - readTensorAddress(cellBuilder); - } - else { // gnarly temporary path to create a type on the fly TODO; Remove when we always have a type - expectObjectStart(buffer.currentToken()); - int initNesting2 = buffer.nesting(); - List> entries = new ArrayList<>(); - for (buffer.next(); buffer.nesting() >= initNesting2; buffer.next()) { - String dimension = buffer.currentName(); - String label = buffer.currentText(); - entries.add(new Pair<>(dimension, label)); - } - TensorType.Builder typeBuilder = new TensorType.Builder(); - for (Pair entry : entries) - typeBuilder.mapped(entry.getFirst()); - tensorBuilder = Tensor.Builder.of(typeBuilder.build()); - cellBuilder = tensorBuilder.cell(); - for (Pair entry : entries) - cellBuilder.label(entry.getFirst(), entry.getSecond()); - expectObjectEnd(buffer.currentToken()); - } + readTensorAddress(cellBuilder); } else if (TENSOR_VALUE.equals(currentName)) { cellValue = Double.valueOf(buffer.currentText()); } } expectObjectEnd(buffer.currentToken()); - if (tensorBuilder == null) { // no content TODO; This will go away with the above - tensorBuilder = Tensor.Builder.of(TensorType.empty); - cellBuilder = tensorBuilder.cell(); - } cellBuilder.value(cellValue); - return tensorBuilder; } private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) { diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java index 52a62caf296..5f1b227790b 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java @@ -9,7 +9,7 @@ import com.yahoo.document.DocumentTypeManager; /** * This interface is used to implement custom deserialization of document updates. * - * @author Ravi Sharma + * @author Ravi Sharma * @author baldersheim */ public interface DocumentReader { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 753008de7e0..6e9495b1437 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -115,8 +115,8 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple // Verify that we have correct version version = getShort(null); if (version < 6 || version > Document.SERIALIZED_VERSION) { - throw new DeserializationException( - "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + "."); + throw new DeserializationException("Unknown version " + version + ", expected " + + Document.SERIALIZED_VERSION + "."); } int dataLength = 0; @@ -278,7 +278,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); - value.assign(TypedBinaryFormat.decode(null, encodedTensor)); // TODO: Pass type + value.assign(TypedBinaryFormat.decode(value.getDataType().getTensorType(), encodedTensor)); } else { value.clear(); } @@ -328,7 +328,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); } - //save a reference to the big buffer we're reading from: + // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; if (version < 7) { diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index 413d1581e58..02773c7dad0 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import java.io.FileOutputStream; import java.io.IOException; @@ -43,6 +44,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { private final String documentId = "doc:something:foooo"; private final String tensorField = "tensorfield"; + private final TensorType tensorType = new TensorType.Builder().mapped("x").build(); private Document createDocument() { return new Document(docMan.getDocumentType("foobar"), new DocumentId(documentId)); @@ -60,7 +62,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { DataType stringwset = DataType.getWeightedSet(DataType.STRING); docType.addField(new Field("strwset", stringwset)); - docType.addField(new Field(tensorField, DataType.TENSOR)); + docType.addField(new Field(tensorField, new TensorDataType(tensorType))); docMan.register(docType); docType2 = new DocumentType("otherdoctype"); @@ -625,7 +627,7 @@ public class DocumentUpdateTestCase extends junit.framework.TestCase { private DocumentUpdate createTensorAssignUpdate() { DocumentUpdate result = new DocumentUpdate(docType, new DocumentId(documentId)); result.addFieldUpdate(FieldUpdate.createAssign(docType.getField(tensorField), - createTensorFieldValue("{{x:0}:2.0}"))); + createTensorFieldValue("{{x:0}:2.0}"))); return result; } diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java index 6f841aac821..6f3abd85521 100644 --- a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java @@ -9,16 +9,17 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; /** - * @author Einar M R Rosenvinge - * @since 5.1.10 + * @author Einar M R Rosenvinge */ public class TemporaryDataTypeTestCase { + @Test public void requireNulls() { - TemporaryDataType type = new TemporaryDataType(0); + TemporaryDataType type = new TemporaryDataType(0, ""); assertThat(type.createFieldValue(new Object()), nullValue()); assertThat(type.createFieldValue(), nullValue()); assertThat(type.getValueClass(), nullValue()); assertThat(type.isValueCompatible(new StringFieldValue("")), is(false)); } + } diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java index 80386141968..c94c917d2ca 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.document.datatypes; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import org.junit.Test; import static org.junit.Assert.assertFalse; @@ -13,8 +14,8 @@ import static org.junit.Assert.assertTrue; */ public class TensorFieldValueTestCase { - private static TensorFieldValue createFieldValue(String tensor) { - return new TensorFieldValue(Tensor.from(tensor)); + private static TensorFieldValue createFieldValue(String tensorString) { + return new TensorFieldValue(Tensor.from(tensorString)); } @Test @@ -22,21 +23,28 @@ public class TensorFieldValueTestCase { assertFalse(createFieldValue("{{x:0}:2.0}").equals(new IntegerFieldValue(5))); } + @Test + public void requireThatDifferentTensorTypesWithEmptyValuesAreNotEqual() { + TensorFieldValue field1 = new TensorFieldValue(new TensorType.Builder().mapped("x").build()); + TensorFieldValue field2 = new TensorFieldValue(new TensorType.Builder().indexed("y").build()); + assertFalse(field1.equals(field2)); + } + @Test public void requireThatDifferentTensorValuesAreNotEqual() { - TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}"); - TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}"); - assertFalse(lhs.equals(rhs)); - assertFalse(lhs.equals(new TensorFieldValue())); + TensorFieldValue field1 = createFieldValue("{{x:0}:2.0}"); + TensorFieldValue field2 = createFieldValue("{{x:0}:3.0}"); + assertFalse(field1.equals(field2)); + assertFalse(field1.equals(new TensorFieldValue(TensorType.empty))); } @Test public void requireThatSameTensorValueIsEqual() { Tensor tensor = Tensor.from("{{x:0}:2.0}"); - TensorFieldValue lhs = new TensorFieldValue(tensor); - TensorFieldValue rhs = new TensorFieldValue(tensor); - assertTrue(lhs.equals(lhs)); - assertTrue(lhs.equals(rhs)); - assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}"))); + TensorFieldValue field1 = new TensorFieldValue(tensor); + TensorFieldValue field2 = new TensorFieldValue(tensor); + assertTrue(field1.equals(field1)); + assertTrue(field1.equals(field2)); + assertTrue(field1.equals(createFieldValue("{{x:0}:2.0}"))); } } diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index ffec7927ab3..a0f993fd2fc 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -10,7 +10,9 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.junit.Test; @@ -27,6 +29,7 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson; */ public class DocumentUpdateJsonSerializerTest { + final static TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build(); final static DocumentTypeManager types = new DocumentTypeManager(); final static JsonFactory parserFactory = new JsonFactory(); final static DocumentType docType = new DocumentType("doctype"); @@ -39,7 +42,7 @@ public class DocumentUpdateJsonSerializerTest { docType.addField(new Field("float_field", DataType.FLOAT)); docType.addField(new Field("double_field", DataType.DOUBLE)); docType.addField(new Field("byte_field", DataType.BYTE)); - docType.addField(new Field("tensor_field", DataType.TENSOR)); + docType.addField(new Field("tensor_field", new TensorDataType(tensorType))); docType.addField(new Field("predicate_field", DataType.PREDICATE)); docType.addField(new Field("raw_field", DataType.RAW)); docType.addField(new Field("int_array", new ArrayDataType(DataType.INT))); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 6c46f743332..34c93580eb3 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -19,6 +19,7 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; @@ -37,7 +38,10 @@ import com.yahoo.document.update.ClearValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.MapValueUpdate; import com.yahoo.document.update.ValueUpdate; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; import org.junit.After; @@ -66,11 +70,12 @@ import static org.junit.Assert.*; /** * Basic test of JSON streams to Vespa document instances. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class JsonReaderTestCase { - DocumentTypeManager types; - JsonFactory parserFactory; + + private DocumentTypeManager types; + private JsonFactory parserFactory; @Rule public ExpectedException exception = ExpectedException.none(); @@ -133,7 +138,10 @@ public class JsonReaderTestCase { } { DocumentType x = new DocumentType("testtensor"); - x.addField(new Field("tensorfield", DataType.TENSOR)); + x.addField(new Field("mappedtensorfield", + new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build()))); + x.addField(new Field("indexedtensorfield", + new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build()))); types.registerDocumentType(x); } { @@ -1022,83 +1030,88 @@ public class JsonReaderTestCase { Document doc = createPutWithoutTensor().getDocument(); assertEquals("testtensor", doc.getId().getDocType()); assertEquals("id:unittest:testtensor::0", doc.getId().toString()); - TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield")); + TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("mappedtensorfield")); assertNull(fieldValue); } @Test public void testParsingOfEmptyTensor() { - assertTensorField("{}", createPutWithTensor("{}")); + assertMappedTensorField("tensor(x{},y{}):{}", createPutWithMappedTensor("{}")); } @Test public void testParsingOfTensorWithEmptyDimensions() { - assertTensorField("{}", - createPutWithTensor("{ " - + " \"dimensions\": [] " - + "}")); + assertMappedTensorField("tensor(x{},y{}):{}", + createPutWithMappedTensor("{ " + + " \"dimensions\": [] " + + "}")); } @Test public void testParsingOfTensorWithEmptyCells() { - assertTensorField("{}", - createPutWithTensor("{ " - + " \"cells\": [] " - + "}")); + assertMappedTensorField("tensor(x{},y{}):{}", + createPutWithMappedTensor("{ " + + " \"cells\": [] " + + "}")); } @Test - public void testParsingOfTensorWithCells() { - assertTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " - + " \"value\": 2.0 }, " - + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, " - + " \"value\": 3.0 } " - + " ]" - + "}")); + public void testParsingOfMappedTensorWithCells() { + Tensor tensor = assertMappedTensorField("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", + createPutWithMappedTensor("{ " + + " \"cells\": [ " + + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " + + " \"value\": 2.0 }, " + + " { \"address\": { \"x\": \"c\", \"y\": \"b\" }, " + + " \"value\": 3.0 } " + + " ]" + + "}")); + assertTrue(tensor instanceof MappedTensor); // any functional instance is fine } @Test - public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() { - assertTensorField("{{x:a,y:b}:2.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"value\": 2.0, " - + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } " - + " ]" - + "}")); + public void testParsingOfIndexedTensorWithCells() { + Tensor tensor = assertTensorField("{{x:0,y:0}:2.0,{x:1,y:0}:3.0}}", + createPutWithTensor("{ " + + " \"cells\": [ " + + " { \"address\": { \"x\": \"0\", \"y\": \"0\" }, " + + " \"value\": 2.0 }, " + + " { \"address\": { \"x\": \"1\", \"y\": \"0\" }, " + + " \"value\": 3.0 } " + + " ]" + + "}", "indexedtensorfield"), "indexedtensorfield"); + assertTrue(tensor instanceof IndexedTensor); // this matters for performance } @Test - public void testParsingOfTensorWithSingleCellWithoutAddress() { - assertTensorField("{{}:2.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"value\": 2.0 } " - + " ]" - + "}")); + public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() { + assertMappedTensorField("{{x:a,y:b}:2.0}", + createPutWithMappedTensor("{ " + + " \"cells\": [ " + + " { \"value\": 2.0, " + + " \"address\": { \"x\": \"a\", \"y\": \"b\" } } " + + " ]" + + "}")); } @Test - public void testParsingOfTensorWithSingleCellWithoutValue() { - assertTensorField("{{x:a}:0.0}", - createPutWithTensor("{ " - + " \"cells\": [ " - + " { \"address\": { \"x\": \"a\" } } " - + " ]" - + "}")); + public void testAssignUpdateOfEmptyMappedTensor() { + assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithMappedTensor("{}")); } @Test - public void testAssignUpdateOfEmptyTensor() { - assertTensorAssignUpdate("{}", createAssignUpdateWithTensor("{}")); + public void testAssignUpdateOfEmptyIndexedTensor() { + try { + assertTensorAssignUpdate("tensor(x{},y{}):{}", createAssignUpdateWithTensor("{}", "indexedtensorfield")); + } + catch (IllegalArgumentException e) { + assertEquals("An indexed tensor must have a value", "Tensor of type tensor(x[],y[]) has no values", e.getMessage()); + } } @Test public void testAssignUpdateOfNullTensor() { - ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithTensor(null)).getValueUpdate(0); + ClearValueUpdate clearUpdate = (ClearValueUpdate) getTensorField(createAssignUpdateWithMappedTensor(null)).getValueUpdate(0); assertTrue(clearUpdate != null); assertTrue(clearUpdate.getValue() == null); } @@ -1106,7 +1119,7 @@ public class JsonReaderTestCase { @Test public void testAssignUpdateOfTensorWithCells() { assertTensorAssignUpdate("{{x:a,y:b}:2.0,{x:c,y:b}:3.0}}", - createAssignUpdateWithTensor("{ " + createAssignUpdateWithMappedTensor("{ " + " \"cells\": [ " + " { \"address\": { \"x\": \"a\", \"y\": \"b\" }, " + " \"value\": 2.0 }, " @@ -1194,10 +1207,13 @@ public class JsonReaderTestCase { return (DocumentPut) reader.next(); } - private DocumentPut createPutWithTensor(String inputTensor) { + private DocumentPut createPutWithMappedTensor(String inputTensor) { + return createPutWithTensor(inputTensor, "mappedtensorfield"); + } + private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[" - + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": " + + " { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": " + inputTensor + " }}" + "]")); @@ -1205,20 +1221,27 @@ public class JsonReaderTestCase { return (DocumentPut) reader.next(); } - private DocumentUpdate createAssignUpdateWithTensor(String inputTensor) { + private DocumentUpdate createAssignUpdateWithMappedTensor(String inputTensor) { + return createAssignUpdateWithTensor(inputTensor, "mappedtensorfield"); + } + private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) { InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {" + Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"" + tensorFieldName + "\": {" + "\"assign\": " + (inputTensor != null ? inputTensor : "null") + " } } } ]")); JsonReader reader = new JsonReader(types, rawDoc, parserFactory); return (DocumentUpdate) reader.next(); } - private static void assertTensorField(String expectedTensor, DocumentPut put) { + private static Tensor assertMappedTensorField(String expectedTensor, DocumentPut put) { + return assertTensorField(expectedTensor, put, "mappedtensorfield"); + } + private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) { final Document doc = put.getDocument(); assertEquals("testtensor", doc.getId().getDocType()); assertEquals(TENSOR_DOC_ID, doc.getId().toString()); - TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield")); + TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField(tensorFieldName)); assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get()); + return fieldValue.getTensor().get(); } private static void assertTensorAssignUpdate(String expectedTensor, DocumentUpdate update) { @@ -1230,7 +1253,7 @@ public class JsonReaderTestCase { } private static FieldUpdate getTensorField(DocumentUpdate update) { - FieldUpdate fieldUpdate = update.getFieldUpdate("tensorfield"); + FieldUpdate fieldUpdate = update.getFieldUpdate("mappedtensorfield"); assertEquals(1, fieldUpdate.size()); return fieldUpdate; } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index f29c208a113..57b3e088efc 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -19,8 +19,10 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; import org.junit.After; @@ -115,7 +117,8 @@ public class JsonWriterTestCase { } { DocumentType x = new DocumentType("testtensor"); - x.addField(new Field("tensorfield", DataType.TENSOR)); + TensorType tensorType = new TensorType.Builder().mapped("x").mapped("y").build(); + x.addField(new Field("tensorfield", new TensorDataType(tensorType))); types.registerDocumentType(x); } } @@ -298,7 +301,7 @@ public class JsonWriterTestCase { @Test public void testWritingOfEmptyTensor() throws IOException { - assertTensorRoundTripEquality("{}","{ \"cells\": [{\"address\": {}, \"value\": 0.0}] }"); + assertTensorRoundTripEquality("{}","{ \"cells\": [] }"); } @Test @@ -320,21 +323,13 @@ public class JsonWriterTestCase { + "}"); } - @Test - public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException { - assertTensorRoundTripEquality("{ " - + " \"cells\": [ " - + " { \"address\": {}, \"value\": 2.0 } " - + " ]" - + "}"); - } - @Test public void testWritingOfTensorFieldValueWithoutTensor() throws IOException { - DocumentType tensorType = types.getDocumentType("testtensor"); + DocumentType documentTypeWithTensor = types.getDocumentType("testtensor"); String docId = "id:unittest:testtensor::0"; - Document doc = new Document(tensorType, docId); - doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue()); + Document doc = new Document(documentTypeWithTensor, docId); + Field tensorField = documentTypeWithTensor.getField("tensorfield"); + doc.setFieldValue(tensorField, new TensorFieldValue(((TensorDataType)tensorField.getDataType()).getTensorType())); assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc)); } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java index f3987085e32..7e3fabc30fb 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java @@ -2,6 +2,7 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.io.GrowableByteBuffer; import java.io.IOException; diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java index 22cb35ae937..ae61bb3cf6f 100644 --- a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java @@ -4,8 +4,10 @@ package com.yahoo.document.serialization; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import org.junit.Test; import java.io.IOException; @@ -19,30 +21,31 @@ import static org.junit.Assert.assertEquals; */ public class TensorFieldValueSerializationTestCase { + private final static TensorType tensorType = new TensorType.Builder().mapped("dimX").mapped("dimY").build(); private final static String TENSOR_FIELD = "my_tensor"; private final static String TENSOR_FILES = "src/test/resources/tensor/"; - private final static TestDocumentFactory docFactory = - new TestDocumentFactory(createDocType(), "id:test:my_type::foo"); + private final static TestDocumentFactory docFactory = new TestDocumentFactory(createDocType(), + "id:test:my_type::foo"); private static DocumentType createDocType() { DocumentType type = new DocumentType("my_type"); - type.addField(TENSOR_FIELD, DataType.TENSOR); + type.addField(TENSOR_FIELD, new TensorDataType(tensorType)); return type; } @Test public void requireThatTensorFieldValueIsSerializedAndDeserialized() { - assertSerialization(new TensorFieldValue()); - assertSerialization(createTensor("{}")); - assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); + assertSerialization(new TensorFieldValue(tensorType)); + assertSerialization(createTensor(tensorType, "{}")); + assertSerialization(createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); } @Test public void requireThatSerializationMatchesCpp() throws IOException { - assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue()); - assertSerializationMatchesCpp("empty_tensor", createTensor("{}")); + assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue(tensorType)); + assertSerializationMatchesCpp("empty_tensor", createTensor(tensorType, "{}")); assertSerializationMatchesCpp("multi_cell_tensor", - createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); + createTensor(tensorType, "{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e,dimY:ff}:5.0}")); } private static void assertSerialization(TensorFieldValue tensor) { @@ -60,8 +63,8 @@ public class TensorFieldValueSerializationTestCase { SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory); } - private static TensorFieldValue createTensor(String tensor) { - return new TensorFieldValue(Tensor.from(tensor)); + private static TensorFieldValue createTensor(TensorType type, String tensorCellString) { + return new TensorFieldValue(Tensor.from(type, tensorCellString)); } } diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java index 3cfbcac5b62..3dc6ebd1403 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java @@ -12,6 +12,7 @@ import com.yahoo.document.predicate.FeatureRange; import com.yahoo.document.predicate.FeatureSet; import com.yahoo.document.predicate.Predicate; import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.tensor.TensorType; import org.apache.commons.codec.binary.Base64; import org.junit.Test; @@ -75,8 +76,8 @@ public class VespaXmlFieldReaderTestCase { @Test public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception { - assertThrows(new Field("my_tensor", DataType.TENSOR), "", - "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); + assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "", + "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); } private class MockedReaderFixture { diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java index 8730265c80d..8a5fabde9ea 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java @@ -7,7 +7,9 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.tensor.TensorType; import org.junit.Ignore; import org.junit.Test; @@ -215,8 +217,8 @@ public class VespaXmlUpdateReaderTestCase { @Test public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception { - assertThrows(new Field("my_tensor", DataType.TENSOR), "", - "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); + assertThrows(new Field("my_tensor", new TensorDataType(TensorType.empty)), "", + "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead."); } private static void assertThrows(Field field, String fieldXml, String expected) throws Exception { diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp index 365182b14eb..2c15c152558 100644 Binary files a/document/src/test/resources/tensor/empty_tensor__cpp and b/document/src/test/resources/tensor/empty_tensor__cpp differ diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java index 1fbade8c785..2c15c152558 100644 Binary files a/document/src/test/resources/tensor/empty_tensor__java and b/document/src/test/resources/tensor/empty_tensor__java differ diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp index c0b2b3a165a..d923fc10559 100644 Binary files a/document/src/test/resources/tensor/multi_cell_tensor__cpp and b/document/src/test/resources/tensor/multi_cell_tensor__cpp differ diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 98aa3f3ae3e..7e5e978f0d1 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -811,7 +811,7 @@ void checkDeserialization(const string &name, std::unique_ptr tensor) { TEST("Require that tensor deserialization matches Java") { checkDeserialization("non_existing_tensor", std::unique_ptr()); - checkDeserialization("empty_tensor", createTensor({{{}, 0.0}}, {})); + checkDeserialization("empty_tensor", createTensor({}, {})); checkDeserialization("multi_cell_tensor", createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 }, {{{"dimX", "ccc"}, diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 267273cfa21..6ababfc19fb 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -60,6 +60,10 @@ datatype[].structtype[].field[].id[].id int ## one specified in config. datatype[].structtype[].field[].datatype int +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +datatype[].structtype[].field[].detailedtype string default="" + ## Specify a document type to inherit datatype[].structtype[].inherits[].name string diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 5e0c5e4e528..21cde02ad54 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -28,7 +28,7 @@ documenttype[].inherits[].id int documenttype[].datatype[].id int ## This is the type of the datatype. -documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE} +documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE, TENSOR} ## This is the id of the datatype of the element in the array. documenttype[].datatype[].array.element.id int default=0 @@ -82,6 +82,10 @@ documenttype[].datatype[].sstruct.field[].id_v6 int ## or one of its inherited document types. documenttype[].datatype[].sstruct.field[].datatype int +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +documenttype[].datatype[].sstruct.field[].detailedtype string default="" + ## The id of the annotation type. documenttype[].annotationtype[].id int diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index ce56597cc86..cd783999456 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -74,10 +74,10 @@ public class ArithmeticExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); - ctx.setValue(evaluate(ctx.setValue(input).execute(lhs).getValue(), - ctx.setValue(input).execute(rhs).getValue())); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); + context.setValue(evaluate(context.setValue(input).execute(lhs).getValue(), + context.setValue(input).execute(rhs).getValue())); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java index d40005bcba1..97034185a40 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -33,8 +33,8 @@ public class Base64DecodeExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java index b29971449e4..dc86c0480ea 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java @@ -25,8 +25,8 @@ public class Base64EncodeExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index fe7771f83ec..79576563504 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -49,17 +49,17 @@ public class CatExpression extends ExpressionList { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); List types = new LinkedList<>(); for (Expression exp : this) { - DataType val = ctx.setValue(input).execute(exp).getValue(); + DataType val = context.setValue(input).execute(exp).getValue(); types.add(val); if (val == null) { throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")."); } } - ctx.setValue(resolveOutputType(types)); + context.setValue(resolveOutputType(types)); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java index 7fb4951bfd9..a8af8f7b4fb 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java @@ -15,8 +15,8 @@ public class ClearStateExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.clear(); + protected void doVerify(VerificationContext context) { + context.clear(); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java index 67f0c2faef4..93fbf4339bb 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java @@ -31,7 +31,7 @@ public class EchoExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java index be70291cb70..9727cc815cf 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -39,7 +39,7 @@ public class ExactExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index f94a100b31d..2ed288760e7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.indexinglanguage.parser.ParseException; import com.yahoo.vespa.objects.Selectable; /** - * @author Simon Thoresen + * @author Simon Thoresen */ public abstract class Expression extends Selectable { @@ -30,9 +30,9 @@ public abstract class Expression extends Selectable { return adapter.getFullOutput(); } - public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate upd) { + public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate update) { DocumentUpdate ret = null; - for (UpdateAdapter adapter : factory.newUpdateAdapterList(upd)) { + for (UpdateAdapter adapter : factory.newUpdateAdapterList(update)) { DocumentUpdate output = adapter.getExpression(expression).execute(adapter); if (output == null) { // ignore @@ -43,7 +43,7 @@ public abstract class Expression extends Selectable { } } if (ret != null) { - ret.setCreateIfNonExistent(upd.getCreateIfNonExistent()); + ret.setCreateIfNonExistent(update.getCreateIfNonExistent()); } return ret; } @@ -57,10 +57,10 @@ public abstract class Expression extends Selectable { return execute(new ExecutionContext(adapter)); } - public final FieldValue execute(ExecutionContext ctx) { + public final FieldValue execute(ExecutionContext context) { DataType inputType = requiredInputType(); if (inputType != null) { - FieldValue input = ctx.getValue(); + FieldValue input = context.getValue(); if (input == null) { return null; } @@ -69,16 +69,16 @@ public abstract class Expression extends Selectable { " input, got " + input.getDataType().getName() + "."); } } - doExecute(ctx); + doExecute(context); DataType outputType = createdOutputType(); if (outputType != null) { - FieldValue output = ctx.getValue(); + FieldValue output = context.getValue(); if (output != null && !outputType.isValueCompatible(output)) { throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() + " output, got " + output.getDataType().getName() + "."); } } - return ctx.getValue(); + return context.getValue(); } protected abstract void doExecute(ExecutionContext ctx); @@ -132,12 +132,10 @@ public abstract class Expression extends Selectable { return verify(new VerificationContext(adapter)); } - public final DataType verify(VerificationContext ctx) { -// System.err.println("enter_verify(exp = '" + this + "', req = " + requiredInputType(ctx) + -// ", in = " + ctx.getValue() + ")"); + public final DataType verify(VerificationContext context) { DataType inputType = requiredInputType(); if (inputType != null) { - DataType input = ctx.getValue(); + DataType input = context.getValue(); if (input == null) { throw new VerificationException(this, "Expected " + inputType.getName() + " input, got null."); } @@ -149,12 +147,10 @@ public abstract class Expression extends Selectable { input.getName() + "."); } } - doVerify(ctx); + doVerify(context); DataType outputType = createdOutputType(); -// System.err.println("exit_verify(exp = '" + this + "', req = " + createdOutputType(ctx) + -// ", out = " + ctx.getValue() + ")"); if (outputType != null) { - DataType output = ctx.getValue(); + DataType output = context.getValue(); if (output == null) { throw new VerificationException(this, "Expected " + outputType.getName() + " output, got null."); } @@ -166,10 +162,10 @@ public abstract class Expression extends Selectable { output.getName() + "."); } } - return ctx.getValue(); + return context.getValue(); } - protected abstract void doVerify(VerificationContext ctx); + protected abstract void doVerify(VerificationContext context); public abstract DataType requiredInputType(); @@ -203,12 +199,13 @@ public abstract class Expression extends Selectable { } return true; } + // Convenience For testing public static Document execute(Expression expression, Document doc) { return expression.execute(new SimpleAdapterFactory(), doc); } - public static final DocumentUpdate execute(Expression expression, DocumentUpdate upd) { - return expression.execute(expression, new SimpleAdapterFactory(), upd); + public static final DocumentUpdate execute(Expression expression, DocumentUpdate update) { + return expression.execute(expression, new SimpleAdapterFactory(), update); } public final FieldValue execute() { return execute(new ExecutionContext()); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java index 915092f5e02..a0dc490d4a4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java @@ -58,8 +58,8 @@ public class FlattenExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index a74f02e9a4a..d634309ca9f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -45,26 +45,26 @@ public class ForEachExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); if (input instanceof ArrayDataType || input instanceof WeightedSetDataType) { - ctx.setValue(((CollectionDataType)input).getNestedType()).execute(exp); + context.setValue(((CollectionDataType)input).getNestedType()).execute(exp); if (input instanceof ArrayDataType) { - ctx.setValue(DataType.getArray(ctx.getValue())); + context.setValue(DataType.getArray(context.getValue())); } else { WeightedSetDataType wset = (WeightedSetDataType)input; - ctx.setValue(DataType.getWeightedSet(ctx.getValue(), wset.createIfNonExistent(), wset.removeIfZero())); + context.setValue(DataType.getWeightedSet(context.getValue(), wset.createIfNonExistent(), wset.removeIfZero())); } } else if (input instanceof StructDataType) { for (Field field : ((StructDataType)input).getFields()) { DataType fieldType = field.getDataType(); - DataType valueType = ctx.setValue(fieldType).execute(exp).getValue(); + DataType valueType = context.setValue(fieldType).execute(exp).getValue(); if (!fieldType.isAssignableFrom(valueType)) { throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " + valueType.getName() + "."); } } - ctx.setValue(input); + context.setValue(input); } else { throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " + input.getName() + "."); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java index 6cc06953437..80fc5ad04f5 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java @@ -38,8 +38,8 @@ public class GetFieldExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); if (!(input instanceof StructuredDataType)) { throw new VerificationException(this, "Expected structured input, got " + input.getName() + "."); } @@ -47,7 +47,7 @@ public class GetFieldExpression extends Expression { if (field == null) { throw new VerificationException(this, "Field '" + fieldName + "' not found."); } - ctx.setValue(field.getDataType()); + context.setValue(field.getDataType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java index 34f0139037b..e18143fb2f4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java @@ -25,12 +25,12 @@ public class GetVarExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getVariable(varName); + protected void doVerify(VerificationContext context) { + DataType input = context.getVariable(varName); if (input == null) { throw new VerificationException(this, "Variable '" + varName + "' not found."); } - ctx.setValue(input); + context.setValue(input); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index b7a49e938b5..db187f0b5d1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -35,8 +35,8 @@ public class GuardExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - exp.verify(ctx); + protected void doVerify(VerificationContext context) { + exp.verify(context); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java index 11e58883b27..b582218f958 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java @@ -37,8 +37,8 @@ public class HexDecodeExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java index 80c7b95308f..18650837ad6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java @@ -18,8 +18,8 @@ public class HexEncodeExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java index 30007090b78..b8fdfb0f26a 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java @@ -22,8 +22,8 @@ public class HostNameExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index eb7e1c9a005..d81672da371 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -95,13 +95,13 @@ public class IfThenExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); - ctx.setValue(input).execute(lhs); - ctx.setValue(input).execute(rhs); - ctx.setValue(input).execute(ifTrue); - ctx.setValue(input).execute(ifFalse); - ctx.setValue(input); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); + context.setValue(input).execute(lhs); + context.setValue(input).execute(rhs); + context.setValue(input).execute(ifTrue); + context.setValue(input).execute(ifFalse); + context.setValue(input); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java index 29957e440a4..618c7787440 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -38,12 +38,12 @@ public class InputExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType val = ctx.getInputType(this, fieldName); + protected void doVerify(VerificationContext context) { + DataType val = context.getInputType(this, fieldName); if (val == null) { throw new VerificationException(this, "Field '" + fieldName + "' not found."); } - ctx.setValue(val); + context.setValue(val); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java index b440c2e68c8..b73ac949cae 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java @@ -44,12 +44,12 @@ public class JoinExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); if (!(input instanceof ArrayDataType)) { throw new VerificationException(this, "Expected Array input, got " + input.getName() + "."); } - ctx.setValue(createdOutputType()); + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java index 272e2c3a3e5..e64c3a21a31 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java @@ -18,8 +18,8 @@ public class LowerCaseExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index 74e2b30a799..441c30dd9d9 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -73,7 +73,7 @@ public class NGramExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java index 75fb355580c..ec1ac5ee4ed 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java @@ -30,8 +30,8 @@ public class NowExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java index 838fb0b3c3b..b3980cf71ad 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java @@ -42,11 +42,11 @@ public class OptimizePredicateExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - checkVariable(ctx, "arity", DataType.INT, true); - checkVariable(ctx, "lower_bound", DataType.LONG, false); - checkVariable(ctx, "upper_bound", DataType.LONG, false); - ctx.setValue(DataType.PREDICATE); + protected void doVerify(VerificationContext context) { + checkVariable(context, "arity", DataType.INT, true); + checkVariable(context, "lower_bound", DataType.LONG, false); + checkVariable(context, "upper_bound", DataType.LONG, false); + context.setValue(DataType.PREDICATE); } private void checkVariable(VerificationContext ctx, String var, DataType type, boolean required) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java index 063ec3a8955..fdda256eab2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java @@ -27,8 +27,8 @@ public abstract class OutputExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.tryOutputType(this, fieldName, ctx.getValue()); + protected void doVerify(VerificationContext context) { + context.tryOutputType(this, fieldName, context.getValue()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 57d833a0748..3b1f3bd234d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -27,8 +27,8 @@ public class ParenthesisExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - innerExp.verify(ctx); + protected void doVerify(VerificationContext context) { + innerExp.verify(context); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java index 8a5cfc1bb88..4aeeafb8ba6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java @@ -38,8 +38,8 @@ public class RandomExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index 79ac7590cac..d536127f123 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -41,12 +41,12 @@ public class ScriptExpression extends ExpressionList { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); for (Expression exp : this) { - ctx.setValue(input).execute(exp); + context.setValue(input).execute(exp); } - ctx.setValue(input); + context.setValue(input); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 8663dcf7a9d..fd079812cfc 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -42,16 +42,16 @@ public class SelectInputExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); for (Pair entry : cases) { - DataType val = ctx.getInputType(this, entry.getFirst()); + DataType val = context.getInputType(this, entry.getFirst()); if (val == null) { throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found."); } - ctx.setValue(val).execute(entry.getSecond()); + context.setValue(val).execute(entry.getSecond()); } - ctx.setValue(input); + context.setValue(input); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java index 35f02ccb05b..59a4d2a42b5 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java @@ -16,7 +16,7 @@ public class SetLanguageExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java index edc1fcfa9ee..59091dc06c0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java @@ -30,8 +30,8 @@ public class SetValueExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(value.getDataType()); + protected void doVerify(VerificationContext context) { + context.setValue(value.getDataType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java index d92831fd6b3..61267f32e33 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java @@ -25,14 +25,14 @@ public class SetVarExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType next = ctx.getValue(); - DataType prev = ctx.getVariable(varName); + protected void doVerify(VerificationContext context) { + DataType next = context.getValue(); + DataType prev = context.getVariable(varName); if (prev != null && !prev.equals(next)) { throw new VerificationException(this, "Attempting to assign conflicting types to variable '" + varName + "', " + prev.getName() + " vs " + next.getName() + "."); } - ctx.setVariable(varName, next); + context.setVariable(varName, next); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java index c2cfcd0cd0e..1fea4029c3b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java @@ -38,8 +38,8 @@ public class SplitExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 012ca87ed37..0595165c5ba 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -35,9 +35,9 @@ public class StatementExpression extends ExpressionList { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { for (Expression exp : this) { - ctx.execute(exp); + context.execute(exp); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java index bb4351bee77..fabfc1f742e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java @@ -44,8 +44,8 @@ public class SubstringExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 21f633c3aba..45ec5c45b74 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -73,8 +73,8 @@ public class SwitchExpression extends CompositeExpression { } @Override - protected void doVerify(VerificationContext ctx) { - DataType input = ctx.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getValue(); if (input == null) { throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got null."); } @@ -83,10 +83,10 @@ public class SwitchExpression extends CompositeExpression { input.getName() + "."); } for (Expression exp : cases.values()) { - ctx.setValue(input).execute(exp); + context.setValue(input).execute(exp); } - ctx.setValue(input).execute(defaultExp); - ctx.setValue(input); + context.setValue(input).execute(defaultExp); + context.setValue(input); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java index 8ad37b98f4c..ffea3f9ac92 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java @@ -15,7 +15,7 @@ public class ThisExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java index 197ed431955..4f072b9c820 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java @@ -26,8 +26,8 @@ public class ToArrayExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(DataType.getArray(ctx.getValue())); + protected void doVerify(VerificationContext context) { + context.setValue(DataType.getArray(context.getValue())); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java index cd01f8251b0..c537fcb5e45 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java @@ -16,8 +16,8 @@ public class ToByteExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java index fb54aefe696..3cf7a1d14d7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java @@ -16,8 +16,8 @@ public class ToDoubleExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java index ebd866abfa9..1bac0df2020 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java @@ -16,8 +16,8 @@ public class ToFloatExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java index 0c900c7756a..b39a3803ba0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java @@ -16,8 +16,8 @@ public class ToIntegerExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java index 63b8b446437..efb914fb203 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java @@ -16,8 +16,8 @@ public class ToLongExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java index 2b89e05b7b2..1b2bb923ae1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java @@ -16,8 +16,8 @@ public class ToPositionExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java index 0db289a6b6b..997bd01646b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java @@ -15,8 +15,8 @@ public class ToStringExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java index f15bac00031..8dcb5336dc7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java @@ -42,8 +42,8 @@ public class ToWsetExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(DataType.getWeightedSet(ctx.getValue(), createIfNonExistent, removeIfZero)); + protected void doVerify(VerificationContext context) { + context.setValue(DataType.getWeightedSet(context.getValue(), createIfNonExistent, removeIfZero)); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java index 272fbde342c..1617b715c68 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -46,7 +46,7 @@ public class TokenizeExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { // empty } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java index d544b9218e4..4b23be7fe91 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java @@ -16,8 +16,8 @@ public class TrimExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java index cdf795ea7d5..65e06c970a8 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java @@ -32,8 +32,8 @@ public class ZCurveExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { - ctx.setValue(createdOutputType()); + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); } @Override diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index cd5f17c3b98..068ebedc4fc 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -264,7 +264,7 @@ public class ExpressionConverterTestCase { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java index 2bf00c98490..a13cc2125e5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java @@ -237,7 +237,7 @@ public class ForEachTestCase { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java index 11f269b4d01..9f3b431ff75 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java @@ -47,9 +47,9 @@ class SimpleExpression extends Expression { } @Override - protected void doVerify(VerificationContext ctx) { + protected void doVerify(VerificationContext context) { if (hasVerifyValue) { - ctx.setValue(verifyValue); + context.setValue(verifyValue); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index fbc469c1829..b0132693fa3 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -74,6 +74,25 @@ public class TensorType { return Optional.empty(); } + /** + * Returns whether a tensor of the given type can be assigned to this type, + * i.e of this type is a generalization of the given type. + */ + public boolean isAssignableTo(TensorType other) { + if (other.dimensions().size() != this.dimensions().size()) return false; + for (int i = 0; i < other.dimensions().size(); i++) { + Dimension thisDimension = this.dimensions().get(i); + Dimension otherDimension = other.dimensions().get(i); + if (thisDimension.isIndexed() != otherDimension.isIndexed()) return false; + if ( ! thisDimension.name().equals(otherDimension.name())) return false; + if (thisDimension.size().isPresent()) { + if ( ! otherDimension.size().isPresent()) return false; + if (otherDimension.size().get() > thisDimension.size().get() ) return false; + } + } + return true; + } + @Override public String toString() { return "tensor(" + dimensions.stream().map(Dimension::toString).collect(Collectors.joining(",")) + ")"; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java index 30b36e83457..8ab23c8d77c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java @@ -54,24 +54,12 @@ class SparseBinaryFormat implements BinaryFormat { @Override public Tensor decode(TensorType type, GrowableByteBuffer buffer) { - if (type == null) // TODO (January 2017): Remove this when types are available - type = decodeDimensionsToType(buffer); - else - consumeAndValidateDimensions(type, buffer); + consumeAndValidateDimensions(type, buffer); Tensor.Builder builder = Tensor.Builder.of(type); decodeCells(buffer, builder, type); return builder.build(); } - private TensorType decodeDimensionsToType(GrowableByteBuffer buffer) { - TensorType.Builder builder = new TensorType.Builder(); - int numDimensions = buffer.getInt1_4Bytes(); - for (int i = 0; i < numDimensions; ++i) { - builder.mapped(buffer.getUtf8String()); - } - return builder.build(); - } - private void consumeAndValidateDimensions(TensorType type, GrowableByteBuffer buffer) { int dimensionCount = buffer.getInt1_4Bytes(); if (type.dimensions().size() != dimensionCount) diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java index 65216aa2fcd..19c1810d928 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -24,7 +24,7 @@ public class TypedBinaryFormat { public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); - if (tensor instanceof IndexedTensor && 1==2) { // TODO: Activate when we have type information everywhere + if (tensor instanceof IndexedTensor) { buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); new DenseBinaryFormat().encode(buffer, tensor); } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java index e0edc6f4e64..d303a69a68d 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -16,7 +16,7 @@ import java.util.HashMap; * methods. * * @author baldersheim - * @author Simon Thoresen + * @author Simon Thoresen */ public class Identifiable extends Selectable implements Cloneable { @@ -177,7 +177,7 @@ public class Identifiable extends Selectable implements Cloneable { * * @param id The class identifier to register with. * @param spec The class to register. - * @return The identifier argument. + * @return the identifier argument. */ protected static int registerClass(int id, Class spec) { if (registry == null) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java index d2b2044f3ed..15e82e6b15c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java @@ -28,7 +28,6 @@ public class DenseBinaryFormatTestCase { } @Test - @Ignore // TODO: Activate when encoding in this format is activated public void requireThatSerializationFormatDoNotChange() { byte[] encodedTensor = new byte[]{2, // binary format type 2, // dimension count -- cgit v1.2.3