diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-12-14 11:28:18 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-12-14 11:28:18 +0100 |
commit | b4e3917d746fcf4be76d353be7c3b4ae9b229973 (patch) | |
tree | 9bbc8f32467adbc022ff1464e0e3514076c79a36 | |
parent | 7b8bb9e7e84c43240457e24bd846291c24221747 (diff) |
Support for configuring and feeding float16 and bool field types
55 files changed, 708 insertions, 326 deletions
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java index cb4b3f6f532..7e5f393c3e0 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java @@ -12,8 +12,8 @@ import java.util.Map; */ public class DataTypeRepo implements DataTypeCollection { - Map<Integer, DataType> typeById = new LinkedHashMap<>(); - Map<String, DataType> typeByName = new LinkedHashMap<>(); + private Map<Integer, DataType> typeById = new LinkedHashMap<>(); + private Map<String, DataType> typeByName = new LinkedHashMap<>(); public DataType getDataType(String name) { return typeByName.get(name); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java index 3e711a48109..debb22ece9e 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java @@ -7,7 +7,9 @@ import java.util.Collection; * @author baldersheim */ public interface DocumentTypeCollection { - public NewDocumentType getDocumentType(NewDocumentType.Name name); - public NewDocumentType getDocumentType(int id); - public Collection<NewDocumentType> getTypes(); + + NewDocumentType getDocumentType(NewDocumentType.Name name); + NewDocumentType getDocumentType(int id); + Collection<NewDocumentType> getTypes(); + } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java index 5138caf0b28..7ab8d0f1d8d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java @@ -10,8 +10,8 @@ import java.util.Map; */ public class DocumentTypeRepo implements DocumentTypeCollection { - final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>(); - final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>(); + private final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>(); + private final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>(); public final NewDocumentType getDocumentType(String name) { return typeByName.get(new NewDocumentType.Name(name)); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 06bb32213f9..fc42864f1d0 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -32,7 +32,6 @@ import static java.util.Collections.emptySet; */ public final class NewDocumentType extends StructuredDataType implements DataTypeCollection { - public static final class Name { // TODO: privatize @@ -381,10 +380,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return this; } - /** - * The field sets defined for this type and its {@link Search} - * @return fieldsets - */ + /** The field sets defined for this type and its {@link Search} */ public Set<FieldSet> getFieldSets() { return Collections.unmodifiableSet(fieldSets); } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java index 2f2c308e633..69fe6f74d27 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java @@ -6,7 +6,7 @@ import com.yahoo.document.DataTypeName; import com.yahoo.document.PositionDataType; /** - * This class represents the builtin 'doument' document type that all other documenttypes inherits. + * This class represents the builtin 'document' document type that all other documenttypes inherits. * Remember that changes here must be compatible. Changes to types of fields can not be done here. * This must also match the mirroring class in c++. * @@ -32,6 +32,8 @@ public class VespaDocumentType { vespa.add(PositionDataType.INSTANCE); vespa.add(DataType.URI); vespa.add(DataType.PREDICATE); + vespa.add(DataType.BOOL); + vespa.add(DataType.FLOAT16); return vespa; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java index bdfebfd0546..4375b446e98 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java @@ -22,10 +22,12 @@ public class SummaryClassField { /** The summary field type enumeration */ public enum Type { + BOOL("bool"), BYTE("byte"), SHORT("short"), INTEGER("integer"), INT64("int64"), + FLOAT16("float16"), FLOAT("float"), DOUBLE("double"), STRING("string"), @@ -77,10 +79,14 @@ public class SummaryClassField { return Type.INTEGER; } else if (fval instanceof LongFieldValue) { return Type.INT64; + } else if (fval instanceof Float16FieldValue) { + return Type.FLOAT16; } else if (fval instanceof FloatFieldValue) { return Type.FLOAT; } else if (fval instanceof DoubleFieldValue) { return Type.DOUBLE; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; } else if (fval instanceof ByteFieldValue) { return Type.BYTE; } else if (fval instanceof Raw) { @@ -102,8 +108,7 @@ public class SummaryClassField { } else if (fieldType instanceof ReferenceDataType) { return Type.LONGSTRING; } else { - throw new IllegalArgumentException("Don't know which summary type to " + - "convert " + fieldType + " to"); + throw new IllegalArgumentException("Don't know which summary type to convert " + fieldType + " to"); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java index ef85c0617bc..3f1474704fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java @@ -1,8 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.searchdefinition.FieldSets; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.FieldSet; @@ -114,14 +122,16 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { /** The streaming field type enumeration */ public static class Type { - public static Type INT8=new Type("int8","INT8"); - public static Type INT16=new Type("int16","INT16"); - public static Type INT32=new Type("int32","INT32"); - public static Type INT64=new Type("int64","INT64"); - public static Type FLOAT=new Type("float","FLOAT"); - public static Type DOUBLE=new Type("double","DOUBLE"); - public static Type STRING=new Type("string","AUTOUTF8"); - public static Type UNSEARCHABLESTRING=new Type("string","NONE"); + public static Type INT8 = new Type("int8","INT8"); + public static Type INT16 = new Type("int16","INT16"); + public static Type INT32 = new Type("int32","INT32"); + public static Type INT64 = new Type("int64","INT64"); + public static Type FLOAT16 = new Type("float16", "FLOAT16"); + public static Type FLOAT = new Type("float","FLOAT"); + public static Type DOUBLE = new Type("double","DOUBLE"); + public static Type STRING = new Type("string","AUTOUTF8"); + public static Type BOOL = new Type("bool","BOOL"); + public static Type UNSEARCHABLESTRING = new Type("string","NONE"); private String name; @@ -148,6 +158,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { return this.name.equals(((Type)other).name); } + @Override public String toString() { return "type: " + name; } @@ -168,18 +179,24 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { /** Converts to the right index type from a field datatype */ private static Type convertType(DataType fieldType) { FieldValue fval = fieldType.createFieldValue(); - if (fieldType.equals(DataType.FLOAT)) { + if (fieldType.equals(DataType.FLOAT16)) { + return Type.FLOAT16; + } else if (fieldType.equals(DataType.FLOAT)) { return Type.FLOAT; } else if (fieldType.equals(DataType.LONG)) { return Type.INT64; } else if (fieldType.equals(DataType.DOUBLE)) { return Type.DOUBLE; + } else if (fieldType.equals(DataType.BOOL)) { + return Type.BOOL; } else if (fieldType.equals(DataType.BYTE)) { return Type.INT8; } else if (fieldType instanceof NumericDataType) { return Type.INT32; } else if (fval instanceof StringFieldValue) { return Type.STRING; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; } else if (fval instanceof Raw) { return Type.STRING; } else if (fval instanceof PredicateFieldValue) { @@ -191,8 +208,8 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } else if (fieldType instanceof ReferenceDataType) { return Type.UNSEARCHABLESTRING; } else { - throw new IllegalArgumentException("Don't know which streaming" + - " field type to " + "convert " + fieldType + " to"); + throw new IllegalArgumentException("Don't know which streaming field type to convert " + + fieldType + " to"); } } @@ -245,6 +262,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } private static class StreamingDocumentType { + private final String name; private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>(); private final Map<String, FieldSet> userFieldSets; @@ -282,4 +300,5 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { fs.addFieldName(fieldName); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index bdd027f4687..fbcaf2a3a80 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -11,6 +11,7 @@ import com.yahoo.document.StructuredDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; import com.yahoo.document.datatypes.FieldValue; @@ -19,6 +20,7 @@ import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.PredicateFieldValue; import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; @@ -82,9 +84,11 @@ public final class Attribute implements Cloneable, Serializable { SHORT("short", "INT16"), INTEGER("integer", "INT32"), LONG("long", "INT64"), + FLOAT16("float16", "FLOAT16"), FLOAT("float", "FLOAT"), DOUBLE("double", "DOUBLE"), STRING("string", "STRING"), + BOOL("bool", "BOOL"), PREDICATE("predicate", "PREDICATE"), TENSOR("tensor", "TENSOR"), REFERENCE("reference", "REFERENCE"); @@ -235,6 +239,10 @@ public final class Attribute implements Cloneable, Serializable { return Type.FLOAT; } else if (fval instanceof DoubleFieldValue) { return Type.DOUBLE; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; + } else if (fval instanceof Float16FieldValue) { + return Type.FLOAT16; } else if (fval instanceof ByteFieldValue) { return Type.BYTE; } else if (fval instanceof Raw) { @@ -288,8 +296,10 @@ public final class Attribute implements Cloneable, Serializable { case STRING : return DataType.STRING; case INTEGER: return DataType.INT; case LONG: return DataType.LONG; + case FLOAT16: return DataType.FLOAT16; case FLOAT: return DataType.FLOAT; case DOUBLE: return DataType.DOUBLE; + case BOOL: return DataType.BOOL; case BYTE: return DataType.BYTE; case PREDICATE: return DataType.PREDICATE; case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d2d28dadfda..16e1e2e4e1d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -146,14 +146,13 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } /** - Creates a new field. - - @param name The name of the field - @param dataType The datatype of the field - @param isHeader Whether this is a "header" field or a "content" field - (true = "header"). - @param owner the owning document (used to check for id collisions) - */ + * Creates a new field. + * + * @param name The name of the field + * @param dataType The datatype of the field + * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). + * @param owner the owning document (used to check for id collisions) + */ protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) { super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; @@ -161,30 +160,29 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } /** - Creates a new field. - - @param name The name of the field - @param dataType The datatype of the field - @param isHeader Whether this is a "header" field or a "content" field - (true = "header"). - @param owner The owning document (used to check for id collisions) - @param fieldMatching The matching object to set for the field - */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { + * Creates a new field. + * + * @param name The name of the field + * @param dataType The datatype of the field + * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). + * @param owner The owning document (used to check for id collisions) + * @param fieldMatching The matching object to set for the field + */ + protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, + Matching fieldMatching, boolean populate, int recursion) { super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; - if (fieldMatching != null) { + if (fieldMatching != null) this.setMatching(fieldMatching); - } populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion); } /** - Constructor for <b>header</b> fields - - @param name The name of the field - @param dataType The datatype of the field - */ + * Constructor for <b>header</b> fields + * + * @param name The name of the field + * @param dataType The datatype of the field + */ public SDField(SDDocumentType repo, String name, DataType dataType) { this(repo, name,dataType,true, true); } @@ -759,6 +757,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** * A list of query commands + * * @return a list of strings with query commands. */ @Override @@ -783,19 +782,18 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return ownerDocType; } - /** - * Two fields are equal if they have the same name - * No, they are not. - */ + @Override public boolean equals(Object other) { if ( ! (other instanceof SDField)) return false; return super.equals(other); } + @Override public int hashCode() { return getName().hashCode(); } + @Override public String toString() { return "field '" + getName() + "'"; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java index dd65cc27625..7e80b967f17 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java @@ -7,7 +7,7 @@ import java.io.Serializable; * A search-time document attribute sort specification(per-document in-memory value). * This belongs to the attribute or field(implicitt attribute). * - * @author baldersheim + * @author baldersheim */ public final class Sorting implements Cloneable, Serializable { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 34a2fc7ab0e..60b3cb6987c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -86,13 +86,13 @@ public abstract class SearchCluster extends AbstractSearchCluster SummaryConfig.Builder summaryConfigBuilder = new SummaryConfig.Builder(); summaryConfigProducer.getConfig(summaryConfigBuilder); - SummaryConfig summaryConfig = new SummaryConfig(summaryConfigBuilder); + SummaryConfig summaryConfig = summaryConfigBuilder.build(); SummarymapConfig summarymapConfig = null; if (summarymapConfigProducer != null) { SummarymapConfig.Builder summarymapConfigBuilder = new SummarymapConfig.Builder(); summarymapConfigProducer.getConfig(summarymapConfigBuilder); - summarymapConfig = new SummarymapConfig(summarymapConfigBuilder); + summarymapConfig = summarymapConfigBuilder.build(); } for (SummaryConfig.Classes sclass : summaryConfig.classes()) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 31f81b76ea4..e50cbabeb9f 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -828,14 +828,13 @@ SDDocumentType structDefinition(Search search, SDDocumentType repo) : */ DataType dataType() : { - String typeName=null; + String typeName = null; boolean isArrayOldStyle = false; - DataType mapType=null; - DataType arrayType=null; - DataType wsetType=null; + DataType mapType = null; + DataType arrayType = null; + DataType wsetType = null; TensorType tensorType; TemporaryStructuredDataType referenceType; - } { ( LOOKAHEAD(<ARRAY> <LESSTHAN>) ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } ) @@ -849,10 +848,9 @@ DataType dataType() : { DataType type = VespaDocumentType.INSTANCE.getDataType(typeName); - //is type still null? if (type == null) { - //we are basically creating TemporaryStructDataType instances for ANYTHING here!! - //we must do this and clean them up later. + // we are basically creating TemporaryStructDataType instances for ANYTHING here!! + // we must do this and clean them up later. type = TemporaryStructuredDataType.create(typeName); } diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index cf3fcebfbcd..e6ffc37e871 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -40,6 +40,48 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].name "abool" +attribute[].datatype BOOL +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "ashortfloat" +attribute[].datatype FLOAT16 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index 647c26e1316..0644659cae7 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -131,6 +131,12 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "along" datatype[].structtype[].field[].datatype 4 datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "abool" +datatype[].structtype[].field[].datatype 6 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "ashortfloat" +datatype[].structtype[].field[].datatype 7 +datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "arrayfield" datatype[].structtype[].field[].datatype -1245117006 datatype[].structtype[].field[].detailedtype "" @@ -232,6 +238,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1328581348 datatype[].documenttype[].bodystruct 348447225 datatype[].documenttype[].fieldsets{[document]}.fields[] "Folders" +datatype[].documenttype[].fieldsets{[document]}.fields[] "abool" datatype[].documenttype[].fieldsets{[document]}.fields[] "abyte" datatype[].documenttype[].fieldsets{[document]}.fields[] "album0" datatype[].documenttype[].fieldsets{[document]}.fields[] "album1" @@ -239,6 +246,7 @@ datatype[].documenttype[].fieldsets{[document]}.fields[] "along" datatype[].documenttype[].fieldsets{[document]}.fields[] "arrarr" datatype[].documenttype[].fieldsets{[document]}.fields[] "arrayfield" datatype[].documenttype[].fieldsets{[document]}.fields[] "arraymapfield" +datatype[].documenttype[].fieldsets{[document]}.fields[] "ashortfloat" datatype[].documenttype[].fieldsets{[document]}.fields[] "complexarray" datatype[].documenttype[].fieldsets{[document]}.fields[] "doublemapfield" datatype[].documenttype[].fieldsets{[document]}.fields[] "floatmapfield" diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 92ca728cea7..3bcdd16e3d6 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -3,6 +3,8 @@ fieldmatchmaxlength 1000000 ilscript[].doctype "types" ilscript[].docfield[] "abyte" ilscript[].docfield[] "along" +ilscript[].docfield[] "abool" +ilscript[].docfield[] "ashortfloat" ilscript[].docfield[] "arrayfield" ilscript[].docfield[] "setfield" ilscript[].docfield[] "setfield2" @@ -30,6 +32,8 @@ ilscript[].docfield[] "complexarray" ilscript[].content[] "clear_state | guard { input along | attribute other; }" ilscript[].content[] "clear_state | guard { input abyte | summary abyte | attribute abyte; }" ilscript[].content[] "clear_state | guard { input along | summary along | attribute along; }" +ilscript[].content[] "clear_state | guard { input abool | summary abool | attribute abool; }" +ilscript[].content[] "clear_state | guard { input ashortfloat | summary ashortfloat | attribute ashortfloat; }" ilscript[].content[] "clear_state | guard { input arrayfield | attribute arrayfield; }" ilscript[].content[] "clear_state | guard { input setfield | attribute setfield; }" ilscript[].content[] "clear_state | guard { input setfield2 | attribute setfield2; }" diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 32375dc09f4..576a95de06f 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -15,6 +15,18 @@ indexinfo[].command[].indexname "along" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "along" indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "abool" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "abool" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "abool" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "ashortfloat" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "ashortfloat" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "ashortfloat" +indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "arrayfield" indexinfo[].command[].command "index" indexinfo[].command[].indexname "arrayfield" diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index 3a73185b325..e5485a24c8c 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -1,10 +1,14 @@ -defaultsummaryid 1103008471 -classes[].id 1103008471 +defaultsummaryid 1131946680 +classes[].id 1131946680 classes[].name "default" classes[].fields[].name "abyte" classes[].fields[].type "byte" classes[].fields[].name "along" classes[].fields[].type "int64" +classes[].fields[].name "abool" +classes[].fields[].type "bool" +classes[].fields[].name "ashortfloat" +classes[].fields[].type "float16" classes[].fields[].name "tagfield" classes[].fields[].type "jsonstring" classes[].fields[].name "stringmapfield" @@ -19,7 +23,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "documentid" classes[].fields[].type "longstring" -classes[].id 278794929 +classes[].id 1027812395 classes[].name "attributeprefetch" classes[].fields[].name "other" classes[].fields[].type "int64" @@ -27,6 +31,10 @@ classes[].fields[].name "abyte" classes[].fields[].type "byte" classes[].fields[].name "along" classes[].fields[].type "int64" +classes[].fields[].name "abool" +classes[].fields[].type "bool" +classes[].fields[].name "ashortfloat" +classes[].fields[].type "float16" classes[].fields[].name "juletre" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/types/summarymap.cfg b/config-model/src/test/derived/types/summarymap.cfg index 0cb8b6129fa..b87200f6573 100644 --- a/config-model/src/test/derived/types/summarymap.cfg +++ b/config-model/src/test/derived/types/summarymap.cfg @@ -5,6 +5,12 @@ override[].arguments "abyte" override[].field "along" override[].command "attribute" override[].arguments "along" +override[].field "abool" +override[].command "attribute" +override[].arguments "abool" +override[].field "ashortfloat" +override[].command "attribute" +override[].arguments "ashortfloat" override[].field "tagfield" override[].command "attribute" override[].arguments "tagfield" diff --git a/config-model/src/test/derived/types/types.sd b/config-model/src/test/derived/types/types.sd index c908b648340..839cb08dbd6 100644 --- a/config-model/src/test/derived/types/types.sd +++ b/config-model/src/test/derived/types/types.sd @@ -11,6 +11,14 @@ search types { indexing: index | summary | attribute } + field abool type bool { + indexing: summary | attribute + } + + field ashortfloat type float16 { + indexing: summary | attribute + } + field arrayfield type array<int> { indexing: attribute } diff --git a/config-model/src/test/derived/types/vsmsummary.cfg b/config-model/src/test/derived/types/vsmsummary.cfg index b1a29b94491..5ee589ccf9f 100644 --- a/config-model/src/test/derived/types/vsmsummary.cfg +++ b/config-model/src/test/derived/types/vsmsummary.cfg @@ -5,6 +5,12 @@ fieldmap[].command NONE fieldmap[].summary "along" fieldmap[].document[].field "along" fieldmap[].command NONE +fieldmap[].summary "abool" +fieldmap[].document[].field "abool" +fieldmap[].command NONE +fieldmap[].summary "ashortfloat" +fieldmap[].document[].field "ashortfloat" +fieldmap[].command NONE fieldmap[].summary "tagfield" fieldmap[].document[].field "tagfield" fieldmap[].command NONE diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 08ab9337e46..344322d3873 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -2,7 +2,7 @@ namespace=vespa.config.search attribute[].name string -attribute[].datatype enum { STRING, UINT1, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE +attribute[].datatype enum { STRING, BOOL, UINT1, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java new file mode 100644 index 00000000000..633e76a2d9c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a byte field in the result set + * + */ + +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.search.result.NanNumber; + +/** + * @author bratseth + */ +public class BoolField extends DocsumField { + + public BoolField(String name) { + super(name); + } + + @Override + public Object convert(Inspector value) { return value.asBool(); } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java index 256edc66793..22069d0270c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java @@ -6,9 +6,6 @@ package com.yahoo.prelude.fastsearch; - -import java.nio.ByteBuffer; - import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 675c1af6545..cdf696ebe1e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -4,11 +4,9 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yahoo.data.access.Inspector; -import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 7d68e7b6679..777919286dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -48,10 +48,12 @@ public abstract class DocsumField { fieldFactory = new FieldFactory(); try { + fieldFactory.put("bool", BoolField.class); fieldFactory.put("byte", ByteField.class); fieldFactory.put("short", ShortField.class); fieldFactory.put("integer", IntegerField.class); fieldFactory.put("int64", Int64Field.class); + fieldFactory.put("float16", Float16Field.class); fieldFactory.put("float", FloatField.class); fieldFactory.put("double", DoubleField.class); fieldFactory.put("string", StringField.class); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java new file mode 100644 index 00000000000..f00d6b19d67 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.search.result.NanNumber; + +/** + * A 16-bit float, represented as a (32-bit) Float in Java, as there is no 16-bit float support. + * + * @author bratseth + */ +public class Float16Field extends DocsumField { + + static final double EMPTY_VALUE = Float.NaN; + + public Float16Field(String name) { + super(name); + } + + private Object convert(float value) { + if (Float.isNaN(value)) { + return NanNumber.NaN; + } else { + return Float.valueOf(value); + } + } + + @Override + public Object convert(Inspector value) { + return convert((float)value.asDouble(EMPTY_VALUE)); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 0ba4133901a..d8b5fc9451a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -304,7 +304,7 @@ public abstract class Item implements Cloneable { public static void putString(String s, ByteBuffer buffer) { putBytes(Utf8.toBytes(s), buffer); } - public static void putBytes(byte [] bytes, ByteBuffer buffer) { + public static void putBytes(byte[] bytes, ByteBuffer buffer) { IntegerCompressor.putCompressedPositiveNumber(bytes.length, buffer); buffer.put(bytes); } @@ -324,6 +324,7 @@ public abstract class Item implements Cloneable { * TODO: Change the output query language into a canonical form of the input * query language */ + @Override public String toString() { StringBuilder buffer = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index e3edf3ea780..764fb2b1118 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -8,12 +8,8 @@ import java.nio.ByteBuffer; /** - * <p>A query term, that is, not only a term in the query language - * (an <i>item</i>), but also a term to be found in (or excluded from) - * the search index.</p> - * - * <p>Optionally, a TermItem may also specify the name of an - * index backend to search.</p> + * Superclass of "leaf" conditions containing a single entity which is either matched in + * a field or not. * * @author bratseth * @author havardpe @@ -84,6 +80,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { */ public boolean isFromQuery() { return isFromQuery; } + @Override public abstract boolean isWords(); /** Sets the origin of this */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java index 246924ea9de..0b3d11158f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java @@ -9,14 +9,14 @@ import com.yahoo.protect.Validator; import java.nio.ByteBuffer; /** - * A term item containing a single word. + * A simple word or token to match in some field. * * @author bratseth * @author havardpe */ public class WordItem extends TermItem { - /** True if this is <b>not</b> part of the special tokens dictionary */ + /** True if this is not part of the special tokens dictionary */ private boolean words = true; /** Is this word stemmed? */ @@ -141,24 +141,20 @@ public class WordItem extends TermItem { } /** Word items uses a empty heading instead of "WORD " */ + @Override protected void appendHeadingString(StringBuilder buffer) {} + @Override public int hashCode() { return word.hashCode() + 71 * super.hashCode(); } + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } + if (!super.equals(object)) return false; WordItem other = (WordItem) object; // Ensured by superclass - - if (!this.word.equals(other.word)) { - return false; - } - - return true; + return this.word.equals(other.word); } public int getNumWords() { @@ -188,4 +184,5 @@ public class WordItem extends TermItem { discloser.addProperty("stemmed", stemmed); discloser.addProperty("words", words); } + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 6eaa1592a3e..5928dd23b28 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -52,6 +52,7 @@ import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.ToolBox; import com.yahoo.prelude.query.ToolBox.QueryVisitor; import com.yahoo.prelude.query.WandItem; @@ -942,13 +943,19 @@ public class YqlParser implements Parser { } @NonNull - private IntItem buildEquals(OperatorNode<ExpressionOperator> ast) { - IntItem number = new IntItem(fetchConditionWord(ast), fetchConditionIndex(ast)); - if (isIndexOnLeftHandSide(ast)) { - return leafStyleSettings(ast.getArgument(1, OperatorNode.class), number); - } else { - return leafStyleSettings(ast.getArgument(0, OperatorNode.class), number); - } + private TermItem buildEquals(OperatorNode<ExpressionOperator> ast) { + String value = fetchConditionWord(ast); + + TermItem item; + if (value.equals("true") || value.equals("false")) + item = new WordItem(value, fetchConditionIndex(ast)); + else + item = new IntItem(value, fetchConditionIndex(ast)); + + if (isIndexOnLeftHandSide(ast)) + return leafStyleSettings(ast.getArgument(1, OperatorNode.class), item); + else + return leafStyleSettings(ast.getArgument(0, OperatorNode.class), item); } @NonNull diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 8c47ae377c3..f2c53fb6b31 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -247,6 +247,12 @@ public class YqlParserTestCase { } @Test + public void testBoolean() { + assertParse("select foo from bar where flag = true;", "flag:true"); + assertParse("select foo from bar where flag = false;", "flag:false"); + } + + @Test public void testTermAnnotations() { assertEquals("merkelapp", getRootWord("select foo from bar where baz contains " + diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index dfe55f5229c..fa5dffd042a 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -3,6 +3,7 @@ package com.yahoo.document; import com.yahoo.collections.Pair; import com.yahoo.concurrent.CopyOnWriteHashMap; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; import com.yahoo.document.datatypes.FieldValue; @@ -11,8 +12,8 @@ import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.PredicateFieldValue; import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.UriFieldValue; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.Identifiable; @@ -39,7 +40,6 @@ public abstract class DataType extends Identifiable implements Serializable, Com // NOTE: These types are also defined in // document/src/vespa/document/datatype/datatype.h // Changes here must also be done there - public final static NumericDataType NONE = new NumericDataType("none", -1, IntegerFieldValue.class, IntegerFieldValue.getFactory()); public final static NumericDataType INT = new NumericDataType("int", 0, IntegerFieldValue.class, IntegerFieldValue.getFactory()); public final static NumericDataType FLOAT = new NumericDataType("float", 1, FloatFieldValue.class, FloatFieldValue.getFactory()); @@ -47,6 +47,8 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory()); public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory()); public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory()); + public final static PrimitiveDataType BOOL = new PrimitiveDataType("bool", 6, BoolFieldValue.class, BoolFieldValue.getFactory()); + public final static NumericDataType FLOAT16 = new NumericDataType("float16", 7, Float16FieldValue.class, Float16FieldValue.getFactory()); public final static DocumentType DOCUMENT = new DocumentType("document"); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); @@ -82,7 +84,6 @@ public abstract class DataType extends Identifiable implements Serializable, Com this.dataTypeId = dataTypeId; } - @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") public DataType clone() { return (DataType)super.clone(); } diff --git a/document/src/main/java/com/yahoo/document/NumericDataType.java b/document/src/main/java/com/yahoo/document/NumericDataType.java index f4a92eca5a1..26da90a709c 100644 --- a/document/src/main/java/com/yahoo/document/NumericDataType.java +++ b/document/src/main/java/com/yahoo/document/NumericDataType.java @@ -4,9 +4,10 @@ package com.yahoo.document; import com.yahoo.vespa.objects.Ids; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class NumericDataType extends PrimitiveDataType { + // The global class identifier shared with C++. public static int classId = registerClass(Ids.document + 52, NumericDataType.class); /** @@ -24,4 +25,5 @@ public class NumericDataType extends PrimitiveDataType { public NumericDataType clone() { return (NumericDataType) super.clone(); } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java new file mode 100644 index 00000000000..2a48b550658 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -0,0 +1,120 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * A boolean field value + * + * @author bratseth + */ +public class BoolFieldValue extends FieldValue { + + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new BoolFieldValue(); + } + } + + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 17, BoolFieldValue.class); + private boolean value; + + public BoolFieldValue() { + this(false); + } + + public BoolFieldValue(boolean value) { + this.value = value; + } + + public BoolFieldValue(String s) { value = Boolean.parseBoolean(s); } + + @Override + public BoolFieldValue clone() { + return (BoolFieldValue)super.clone(); + } + + @Override + public void clear() { + value = false; + } + + @Override + public void assign(Object o) { + if ( ! checkAssign(o)) return; + if (o instanceof String || o instanceof StringFieldValue) { + value = Boolean.parseBoolean(o.toString()); + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public boolean getBoolean() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.BOOL; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printBoolXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + return super.hashCode() + ( value ? 3 : 0); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof BoolFieldValue)) return false; + if ( ! super.equals(o)) return false; + + BoolFieldValue that = (BoolFieldValue) o; + if (value != that.value) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue other) { + int comp = super.compareTo(other); + if (comp != 0) return comp; + return Boolean.compare(value, ((BoolFieldValue)other).value); + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java index 4aa4c29db48..2f8cb96219c 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java @@ -11,16 +11,18 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.objects.Ids; /** - * FieldValue which encapsulates a byte. + * A byte field value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class ByteFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new ByteFieldValue(); } } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } public static final int classId = registerClass(Ids.document + 10, ByteFieldValue.class); private byte value; @@ -125,9 +127,8 @@ public class ByteFieldValue extends NumericFieldValue { } /* (non-Javadoc) - * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) - */ - + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ @Override public void deserialize(Field field, FieldReader reader) { reader.read(field, this); @@ -151,4 +152,5 @@ public class ByteFieldValue extends NumericFieldValue { return 0; } } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java index fe135ccfd05..ad6d329cbae 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java @@ -7,9 +7,9 @@ import java.util.Collection; import java.util.Iterator; /** - * Date: Apr 16, 2008 + * Superclass of multivalue field values * - * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + * @author HÃ¥kon Humberset */ public abstract class CollectionFieldValue<T extends FieldValue> extends CompositeFieldValue { @@ -79,4 +79,5 @@ public abstract class CollectionFieldValue<T extends FieldValue> extends Composi } public abstract int size(); + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java index 3202c6b40af..6efd5a86f75 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java @@ -4,6 +4,7 @@ package com.yahoo.document.datatypes; import com.yahoo.document.DataType; public abstract class CompositeFieldValue extends FieldValue { + private DataType dataType; public CompositeFieldValue(DataType dataType) { @@ -36,4 +37,5 @@ public abstract class CompositeFieldValue extends FieldValue { result = 31 * result + (dataType != null ? dataType.hashCode() : 0); return result; } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java index 0f1fe50818b..99bac017b78 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java @@ -11,16 +11,18 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.objects.Ids; /** - * FieldValue which encapsulates a double. + * A 64-bit float field value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class DoubleFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new DoubleFieldValue(); } } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } public static final int classId = registerClass(Ids.document + 14, DoubleFieldValue.class); private double value; diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java index de645aff297..9189a97dbf6 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.datatypes; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; /** - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public abstract class FieldPathIteratorHandler { @@ -110,4 +109,5 @@ public abstract class FieldPathIteratorHandler { public VariableMap getVariables() { return variables; } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java index 0460e303426..dc3fd36b367 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -31,6 +31,7 @@ public abstract class FieldValue extends Identifiable implements Comparable<Fiel /** * Get XML representation of a single field and all its children, if any. + * * @return XML representation of field in a <value> element */ public String toXml() { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Float16FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/Float16FieldValue.java new file mode 100644 index 00000000000..013f656b7b5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/Float16FieldValue.java @@ -0,0 +1,137 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * A 16-bit float field value + * + * @author bratseth + */ +public final class Float16FieldValue extends NumericFieldValue { + + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new Float16FieldValue(); + } + } + + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 18, Float16FieldValue.class); + private float value; // 16-bit not supported in Java yet + + public Float16FieldValue() { + this((float) 0); + } + + public Float16FieldValue(float value) { + this.value = value; + } + + public Float16FieldValue(Float value) { + this.value = value; + } + + public Float16FieldValue(String s) { value = Float.parseFloat(s); } + + @Override + public Float16FieldValue clone() { + Float16FieldValue val = (Float16FieldValue) super.clone(); + val.value = value; + return val; + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void clear() { + value = 0.0f; + } + + @Override + public void assign(Object obj) { + if (!checkAssign(obj)) return; + + if (obj instanceof Number) + value = ((Number) obj).floatValue(); + else if (obj instanceof NumericFieldValue) + value = (((NumericFieldValue) obj).getNumber().floatValue()); + else if (obj instanceof String || obj instanceof StringFieldValue) + value = Float.parseFloat(obj.toString()); + else + throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + + public float getFloat() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.FLOAT16; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printShortfloatXml(this, xml); + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (value != +0.0f ? Float.floatToIntBits(value) : 0); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Float16FieldValue)) return false; + if (!super.equals(o)) return false; + + Float16FieldValue that = (Float16FieldValue) o; + if (Float.compare(that.value, value) != 0) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + if (comp != 0) return comp; + return Float.compare(value, ((Float16FieldValue) fieldValue).value); + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java index 5a94bb43a77..e09cbea861c 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java @@ -11,16 +11,18 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.objects.Ids; /** - * FieldValue which encapsulates a float. + * A 32-bit float field value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class FloatFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new FloatFieldValue(); } } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } public static final int classId = registerClass(Ids.document + 13, FloatFieldValue.class); private float value; @@ -120,9 +122,8 @@ public final class FloatFieldValue extends NumericFieldValue { } /* (non-Javadoc) - * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) - */ - + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ @Override public void deserialize(Field field, FieldReader reader) { reader.read(field, this); diff --git a/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java index 19f34acde9a..62090bab8cb 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java @@ -11,9 +11,9 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.objects.Ids; /** - * FieldValue which encapsulates an int. + * A 32-bit integer field value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class IntegerFieldValue extends NumericFieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java index 2d3797f6735..45d044bcb9a 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java @@ -11,11 +11,12 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.objects.Ids; /** - * FieldValue which encapsulates a long. + * A 64-bit integer field value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class LongFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new LongFieldValue(); @@ -148,4 +149,5 @@ public final class LongFieldValue extends NumericFieldValue { return 0; } } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Raw.java b/document/src/main/java/com/yahoo/document/datatypes/Raw.java index 23ed0cee23e..7900615aa93 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Raw.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Raw.java @@ -15,7 +15,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; /** - * FieldValue which encapsulates a Raw value + * A field value which is an array of byte data * * @author Einar M R Rosenvinge */ diff --git a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java index 0097e5c93d0..034167b1121 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java @@ -14,21 +14,20 @@ import java.util.Objects; import java.util.Optional; /** - * A reference field value allows search queries to access fields in other document instances + * <p>A reference field value allows search queries to access fields in other document instances * as if they were fields natively stored within the searched document. This allows modelling * one-to-many relations such as a parent document with many children containing references - * back to the parent. + * back to the parent.</p> * - * Each <code>ReferenceFieldValue</code> may contain a single document ID which specifies the + * <p>Each <code>ReferenceFieldValue</code> may contain a single document ID which specifies the * instance the field should refer to. This document ID must have a type matching that of the - * reference data type of the field itself. + * reference data type of the field itself.</p> * - * Note that references are not polymorphic. This means that if you have a document type + * <p>Note that references are not polymorphic. This means that if you have a document type * "foo" inheriting "bar", you cannot have a <code>reference<bar></code> field containing - * a document ID for a "foo" document. + * a document ID for a "foo" document.</p> * * @author vekterli - * @since 6.65 */ public class ReferenceFieldValue extends FieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index 7d4e615b27b..fc75870bb94 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -12,9 +12,7 @@ import com.yahoo.vespa.objects.Ids; import java.util.*; /** - * Date: Apr 15, 2008 - * - * @author humbe + * @author HÃ¥kon Humberset */ public class Struct extends StructuredFieldValue { diff --git a/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java index 13688c8e311..1b51d6625e2 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java @@ -10,12 +10,10 @@ import com.yahoo.net.Url; import java.net.URI; /** - * Created with IntelliJ IDEA. - * User: magnarn - * Date: 11/2/12 - * Time: 2:37 PM + * @author Magnar Nedland */ public class UriFieldValue extends StringFieldValue { + public static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new UriFieldValue(); @@ -46,4 +44,5 @@ public class UriFieldValue extends StringFieldValue { super.deserialize(field, reader); Url.fromString(toString()); // Throws if value is invalid. } + } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 941dbc8d406..a0e7f81ea73 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -7,7 +7,6 @@ import com.yahoo.document.datatypes.*; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; -import java.io.UnsupportedEncodingException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -67,6 +66,14 @@ public class XmlSerializationHelper { xml.addContent(f.toString()); } + public static void printShortfloatXml(Float16FieldValue f, XmlStream xml) { + xml.addContent(f.toString()); + } + + public static void printBoolXml(BoolFieldValue f, XmlStream xml) { + xml.addContent(f.toString()); + } + public static void printIntegerXml(IntegerFieldValue f, XmlStream xml) { xml.addContent(f.toString()); } diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java index ffc372d4851..e9fce35ed01 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java @@ -20,9 +20,6 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; -import java.util.Collection; -import java.util.Iterator; -import java.util.ListIterator; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -31,12 +28,12 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class StringTestCase extends AbstractTypesTest { @Test - public void testDeserialize() throws Exception { + public void testDeserialize() { byte[] buf = new byte[1500]; GrowableByteBuffer data = GrowableByteBuffer.wrap(buf); //short string @@ -122,17 +119,8 @@ public class StringTestCase extends AbstractTypesTest { assertTrue(blah.equals(blah2)); } -/* public void testSpanTree() { - StringFieldValue annotatedText = new StringFieldValue("banana airlines"); - SpanList lingTree = new SpanList(); - annotatedText.setSpanTree("linguistics", lingTree); - for (Annotation anAnnotation : annotatedText.getSpanTree("linguistics")) { - System.err.println(anAnnotation); - } - }*/ - @Test - public void testSerializeDeserialize() throws Exception { + public void testSerializeDeserialize() { java.lang.String test = "Hello hello"; BufferSerializer data = new BufferSerializer(new GrowableByteBuffer(100, 2.0f)); StringFieldValue value = new StringFieldValue(test); @@ -171,7 +159,7 @@ public class StringTestCase extends AbstractTypesTest { } @Test - public void testNestedSpanTreeBug4187377() { + public void testNestedSpanTree() { AnnotationType type = new AnnotationType("ann", DataType.STRING); StringFieldValue outerString = new StringFieldValue("Ballooo"); @@ -224,8 +212,6 @@ public class StringTestCase extends AbstractTypesTest { doc = annotate(doc, manager); doc = serializeAndDeserialize(doc, manager); - doc = consume(doc, manager); - System.err.println(doc); } private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) { @@ -247,11 +233,6 @@ public class StringTestCase extends AbstractTypesTest { AnnotationType location = registry.getType("location"); Map<String, AnnotationType> m = registry.getTypes(); - for (String key : m.keySet()) { - System.out.println("Key: " + key); - AnnotationType val = m.get(key); - parseAnnotationType(val); - } SpanTree tree = new SpanTree("testannotations"); SpanList root = (SpanList)tree.getRoot(); @@ -335,7 +316,7 @@ public class StringTestCase extends AbstractTypesTest { root.remove(branch); tree.cleanup(); - System.out.println("No. Of Annotations: " + tree.numAnnotations()); + assertEquals(5, tree.numAnnotations()); body.setSpanTree(tree); document.setFieldValue(document.getField("body"), body); @@ -343,93 +324,4 @@ public class StringTestCase extends AbstractTypesTest { return document; } - @SuppressWarnings("deprecation") - public Document consume(Document document, DocumentTypeManager docTypeMgr) { - DocumentType type = docTypeMgr.getDocumentType("blog"); - Collection<Field> fc = type.getFields(); - for (Field f : fc) { - System.out.println("\n\nField Name: " + f.getName()); - System.out.println("DataType: " + f.getDataType()); - System.out.println("isHeader? " + f.isHeader()); - - FieldValue val = document.getFieldValue(f); - if (val instanceof StringFieldValue) { - StringFieldValue sfv = (StringFieldValue)val; - System.out.println(f.getName() + " is a StringField. Field Value: " + sfv.getString()); - Collection<SpanTree> c = sfv.getSpanTrees(); - for (SpanTree tree : c) { - System.out.println(f.getName() + " has annotations"); - consumeAnnotations(tree, (SpanList)tree.getRoot()); - } - } - } - - return document; - } - - public void consumeAnnotations(SpanTree tree, SpanList root) { - System.out.println("\n\nSpanList: " + root + " num Children: " + root.numChildren()); - System.out.println("-------------------"); - Iterator<SpanNode> childIterator = root.childIterator(); - while (childIterator.hasNext()) { - SpanNode node = childIterator.next(); - System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr)); - if (node instanceof SpanList) { - System.out.println("Encountered another span list"); - SpanList spl = (SpanList)node; - ListIterator<SpanNode> lli = spl.childIterator(); - while (lli.hasNext()) { - System.out.print(" " + lli.next() + " "); - } - consumeAnnotations(tree, (SpanList)node); - } else { - System.out.println("\nGetting annotations for this span node: " + node); - getAnnotationsForNode(tree, node); - } - } - System.out.println("\nGetting annotations for the SpanList itself : " + root); - getAnnotationsForNode(tree, root); - } - - public void getAnnotationsForNode(SpanTree tree, SpanNode node) { - Iterator<Annotation> iter = tree.iterator(node); - boolean annotationPresent = false; - while (iter.hasNext()) { - annotationPresent = true; - Annotation xx = iter.next(); - Struct fValue = (Struct)xx.getFieldValue(); - System.out.println("Annotation: " + xx); - if (fValue == null) { - System.out.println("Field Value is null"); - return; - } - Iterator fieldIter = fValue.iterator(); - while (fieldIter.hasNext()) { - Map.Entry m = (Map.Entry)fieldIter.next(); - Field f = (Field)m.getKey(); - FieldValue val = (FieldValue)m.getValue(); - System.out.println("Field : " + f + " Value: " + val); - } - } - if (!annotationPresent) { - System.out.println("****No annotations found for the span node: " + node); - } - } - - public void parseAnnotationType(AnnotationType t) { - System.out.println("Type Name: " + t.getName()); - System.out.println("Type ID: " + t.getId()); - DataType dt = t.getDataType(); - String dataTypeStr; - if (dt == DataType.STRING) { - dataTypeStr = "String"; - } else if (dt == DataType.INT) { - dataTypeStr = "Integer"; - } else if (dt == DataType.URI) { - dataTypeStr = "URL"; - } else { - dataTypeStr = "UNKNOWN"; - } - System.out.println("Type DataType: " + dataTypeStr); - } } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 32f63e6c0b3..b3d502ea56f 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.MapFieldValue; @@ -97,6 +98,7 @@ public class JsonReaderTestCase { x.addField(new Field("something", DataType.STRING)); x.addField(new Field("nalle", DataType.STRING)); x.addField(new Field("int1", DataType.INT)); + x.addField(new Field("flag", DataType.BOOL)); types.registerDocumentType(x); } { @@ -172,13 +174,20 @@ public class JsonReaderTestCase { } @Test - public final void readSingleDocumentPut() { - InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," - + " \"fields\": { \"something\": \"smoketest\"," - + " \"nalle\": \"bamse\"}}")); + public void readSingleDocumentPut() { + String doc = + "{ \"put\": \"id:unittest:smoke::doc1\"," + + " \"fields\": { " + + " \"something\": \"smoketest\"," + + " \"flag\": true," + + " \"nalle\": \"bamse\"" + + " } " + + "}"; + + InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(doc)); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentParser.SupportedOperation.PUT, "id:unittest:smoke::whee"); + DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentParser.SupportedOperation.PUT, + "id:unittest:smoke::doc1"); smokeTestDoc(put.getDocument()); } @@ -196,7 +205,7 @@ public class JsonReaderTestCase { } @Test - public final void readClearField() { + public void readClearField() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"int1\": {" @@ -211,11 +220,17 @@ public class JsonReaderTestCase { @Test - public final void smokeTest() throws IOException { - InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," - + " \"fields\": { \"something\": \"smoketest\"," - + " \"nalle\": \"bamse\"}}")); + public void smokeTest() throws IOException { + String doc = + "{ \"put\": \"id:unittest:smoke::doc1\"," + + " \"fields\": { " + + " \"something\": \"smoketest\"," + + " \"flag\": true," + + " \"nalle\": \"bamse\"" + + " } " + + "}"; + + InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(doc)); JsonReader r = new JsonReader(types, rawDoc, parserFactory); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); @@ -225,13 +240,18 @@ public class JsonReaderTestCase { } @Test - public final void docIdLookaheadTest() throws IOException { - InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("{" - + " \"fields\": { \"something\": \"smoketest\"," - + " \"nalle\": \"bamse\"}," - + "\"put\": \"id:unittest:smoke::whee\"" - + "}")); + public void docIdLookaheadTest() throws IOException { + String doc = + "{ \"fields\": { " + + " \"something\": \"smoketest\"," + + " \"flag\": true," + + " \"nalle\": \"bamse\"" + + " }," + + " \"put\": \"id:unittest:smoke::doc1\"" + + " } " + + "}"; + + InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(doc)); JsonReader r = new JsonReader(types, rawDoc, parserFactory); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); @@ -242,10 +262,10 @@ public class JsonReaderTestCase { @Test - public final void emptyDocTest() throws IOException { + public void emptyDocTest() throws IOException { InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," - + " \"fields\": {}}")); + Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," + + " \"fields\": {}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); @@ -255,7 +275,7 @@ public class JsonReaderTestCase { } @Test - public final void testStruct() throws IOException { + public void testStruct() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:mirrors::whee\"," + " \"fields\": { " @@ -285,7 +305,7 @@ public class JsonReaderTestCase { } @Test - public final void testStructUpdate() throws IOException { + public void testStructUpdate() throws IOException { DocumentUpdate put = parseUpdate("{\"update\": \"id:unittest:mirrors:g=test:whee\"," + "\"create\": true," + " \"fields\": { " @@ -331,7 +351,7 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateArray() throws IOException { + public void testUpdateArray() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testarray::whee\"," + " \"fields\": { " + "\"actualarray\": {" + " \"add\": [" @@ -341,7 +361,7 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateWeighted() throws IOException { + public void testUpdateWeighted() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" + " \"add\": {" @@ -365,7 +385,7 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateMatch() throws IOException { + public void testUpdateMatch() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" + " \"match\": {" @@ -391,7 +411,7 @@ public class JsonReaderTestCase { @SuppressWarnings({ "cast", "unchecked", "rawtypes" }) @Test - public final void testArithmeticOperators() throws IOException { + public void testArithmeticOperators() throws IOException { Tuple2[] operations = new Tuple2[] { new Tuple2<String, Operator>(UPDATE_DECREMENT, ArithmeticValueUpdate.Operator.SUB), @@ -428,7 +448,7 @@ public class JsonReaderTestCase { @SuppressWarnings("rawtypes") @Test - public final void testArrayIndexing() throws IOException { + public void testArrayIndexing() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testarray::whee\"," + " \"fields\": { " + "\"actualarray\": {" + " \"match\": {" @@ -451,7 +471,7 @@ public class JsonReaderTestCase { } @Test - public final void testDocumentRemove() { + public void testDocumentRemove() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"remove\": \"id:unittest:smoke::whee\"" + " }}")); @@ -461,7 +481,7 @@ public class JsonReaderTestCase { } @Test - public final void testWeightedSet() throws IOException { + public void testWeightedSet() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testset::whee\"," + " \"fields\": { \"actualset\": {" @@ -482,7 +502,7 @@ public class JsonReaderTestCase { } @Test - public final void testArray() throws IOException { + public void testArray() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testarray::whee\"," + " \"fields\": { \"actualarray\": [" @@ -503,7 +523,7 @@ public class JsonReaderTestCase { } @Test - public final void testMap() throws IOException { + public void testMap() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\"," + " \"fields\": { \"actualmap\": {" @@ -523,7 +543,7 @@ public class JsonReaderTestCase { } @Test - public final void testOldMap() throws IOException { + public void testOldMap() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\"," + " \"fields\": { \"actualmap\": [" @@ -544,7 +564,7 @@ public class JsonReaderTestCase { } @Test - public final void testPositionPositive() throws IOException { + public void testPositionPositive() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\"," + " \"fields\": { \"singlepos\": \"N63.429722;E10.393333\" }}")); @@ -561,7 +581,7 @@ public class JsonReaderTestCase { } @Test - public final void testPositionNegative() throws IOException { + public void testPositionNegative() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\"," + " \"fields\": { \"singlepos\": \"W46.63;S23.55\" }}")); @@ -578,7 +598,7 @@ public class JsonReaderTestCase { } @Test - public final void testRaw() throws IOException { + public void testRaw() throws IOException { String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest")))); InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testraw::whee\"," @@ -600,7 +620,7 @@ public class JsonReaderTestCase { } @Test - public final void testMapStringToArrayOfInt() throws IOException { + public void testMapStringToArrayOfInt() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": { \"bamse\": [1, 2, 3] }}}")); @@ -621,7 +641,7 @@ public class JsonReaderTestCase { } @Test - public final void testOldMapStringToArrayOfInt() throws IOException { + public void testOldMapStringToArrayOfInt() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": [" @@ -644,7 +664,7 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToString() throws IOException { + public void testAssignToString() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": {" + " \"assign\": \"orOther\" }}" + " }"); @@ -655,7 +675,7 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToArray() throws IOException { + public void testAssignToArray() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": {" + " \"assign\": { \"bamse\": [1, 2, 3] }}}}"); @@ -671,7 +691,7 @@ public class JsonReaderTestCase { } @Test - public final void testOldAssignToArray() throws IOException { + public void testOldAssignToArray() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": {" + " \"assign\": [" @@ -689,7 +709,7 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToWeightedSet() throws IOException { + public void testAssignToWeightedSet() throws IOException { DocumentUpdate doc = parseUpdate("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" + " \"assign\": {" @@ -706,10 +726,11 @@ public class JsonReaderTestCase { @Test - public final void testCompleteFeed() { + public void testCompleteFeed() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": \"smoketest\"," + + " \"flag\": true," + " \"nalle\": \"bamse\"}}" + ", " + "{\"update\": \"id:unittest:testarray::whee\"," + " \"fields\": { " + "\"actualarray\": {" @@ -722,10 +743,11 @@ public class JsonReaderTestCase { } @Test - public final void testCompleteFeedWithCreateAndCondition() { + public void testCompleteFeedWithCreateAndCondition() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": \"smoketest\"," + + " \"flag\": true," + " \"nalle\": \"bamse\"}}" + ", " + "{" + "\"condition\":\"bla\"," @@ -755,14 +777,14 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateWithConditionAndCreateInDifferentOrdering() { - final int documentsCreated = 106; + public void testUpdateWithConditionAndCreateInDifferentOrdering() { + int documentsCreated = 106; List<String> parts = Arrays.asList( "\"condition\":\"bla\"", "\"update\": \"id:unittest:testarray::whee\"", " \"fields\": { " + "\"actualarray\": { \"add\": [" + " \"person\",\"another person\"]}}", " \"create\":true"); - final Random random = new Random(42); + Random random = new Random(42); StringBuilder documents = new StringBuilder("["); for (int x = 0; x < documentsCreated; x++) { Collections.shuffle(parts, random); @@ -790,7 +812,7 @@ public class JsonReaderTestCase { @Test(expected=RuntimeException.class) - public final void testCreateIfNonExistentInPut() { + public void testCreateIfNonExistentInPut() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[{" + " \"create\":true," @@ -803,10 +825,11 @@ public class JsonReaderTestCase { } @Test - public final void testCompleteFeedWithIdAfterFields() { + public void testCompleteFeedWithIdAfterFields() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[{" + " \"fields\": { \"something\": \"smoketest\"," + + " \"flag\": true," + " \"nalle\": \"bamse\"}," + "\"put\": \"id:unittest:smoke::whee\"" + "}" + ", " @@ -840,7 +863,7 @@ public class JsonReaderTestCase { @Test - public final void testCompleteFeedWithEmptyDoc() { + public void testCompleteFeedWithEmptyDoc() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": {}}" + ", " @@ -879,10 +902,13 @@ public class JsonReaderTestCase { } private void smokeTestDoc(Document doc) { - FieldValue f = doc.getFieldValue(doc.getField("nalle")); - assertSame(StringFieldValue.class, f.getClass()); - StringFieldValue s = (StringFieldValue) f; - assertEquals("bamse", s.getString()); + FieldValue boolField = doc.getFieldValue(doc.getField("flag")); + assertSame(BoolFieldValue.class, boolField.getClass()); + assertTrue((Boolean)boolField.getWrappedValue()); + + FieldValue stringField = doc.getFieldValue(doc.getField("nalle")); + assertSame(StringFieldValue.class, stringField.getClass()); + assertEquals("bamse", ((StringFieldValue) stringField).getString()); } @Test @@ -901,7 +927,7 @@ public class JsonReaderTestCase { } @Test - public final void feedWithBasicErrorTest() { + public void feedWithBasicErrorTest() { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("[" + " { \"put\": \"id:test:smoke::0\", \"fields\": { \"something\": \"foo\" } }," @@ -915,10 +941,11 @@ public class JsonReaderTestCase { } @Test - public final void idAsAliasForPutTest() throws IOException{ + public void idAsAliasForPutTest() throws IOException{ InputStream rawDoc = new ByteArrayInputStream( - Utf8.toBytes("{\"id\": \"id:unittest:smoke::whee\"," + Utf8.toBytes("{\"id\": \"id:unittest:smoke::doc1\"," + " \"fields\": { \"something\": \"smoketest\"," + + " \"flag\": true," + " \"nalle\": \"bamse\"}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); DocumentParseInfo parseInfo = r.parseDocument().get(); @@ -948,7 +975,7 @@ public class JsonReaderTestCase { } @Test - public final void testFeedWithTestAndSetConditionOrderingOne() { + public void testFeedWithTestAndSetConditionOrderingOne() { testFeedWithTestAndSetCondition( inputJson("[", " {", diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index 1630d267302..6f40b590a64 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -109,12 +109,13 @@ public final class Utf8 { /** * Will try an optimistic approach to utf8 encoding. * That is 4.6x faster that the brute encode for ascii, not accounting for reduced memory footprint and GC. - * @param str The string to encode. + * + * @param string The string to encode. * @return Utf8 encoded array */ - public static byte[] toBytes(String str) { - byte [] utf8 = toBytesAscii(str); - return utf8 != null ? utf8 : str.getBytes(StandardCharsets.UTF_8); + public static byte[] toBytes(String string) { + byte [] utf8 = toBytesAscii(string); + return utf8 != null ? utf8 : string.getBytes(StandardCharsets.UTF_8); } /** * Will try an optimistic approach to utf8 decoding. diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java index 0aee5fcdea5..3e392448ce2 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -175,9 +175,9 @@ public class Identifiable extends Selectable implements Cloneable { * * <code>public static int classId = registerClass(<id>, <ClassName>.class);</code> * - * @param id The class identifier to register with. - * @param spec The class to register. - * @return the identifier argument. + * @param id the class identifier to register with + * @param spec the class to register + * @return the identifier argument */ protected static int registerClass(int id, Class<? extends Identifiable> spec) { if (registry == null) { diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java index 64a62ae37f3..02ae6a58bf7 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java @@ -8,8 +8,10 @@ package com.yahoo.vespa.objects; * @author baldersheim */ public interface Ids { - public static int document = 0x1000; - public static int searchlib = 0x4000; - public static int vespa_configmodel = 0x7000; - public static int annotation = 0x10000; + + int document = 0x1000; + int searchlib = 0x4000; + int vespa_configmodel = 0x7000; + int annotation = 0x10000; + } diff --git a/vsm/src/vespa/vsm/config/vsmfields.def b/vsm/src/vespa/vsm/config/vsmfields.def index 0b17875af69..ca48692b526 100644 --- a/vsm/src/vespa/vsm/config/vsmfields.def +++ b/vsm/src/vespa/vsm/config/vsmfields.def @@ -12,7 +12,7 @@ searchall int default=1 fieldspec[].name string ## The search method for a given field. Note: same field in 2 different document types must match on type if not a random result might be expected. -fieldspec[].searchmethod enum { NONE, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT, DOUBLE } default=AUTOUTF8 +fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE } default=AUTOUTF8 fieldspec[].arg1 string default="" ## Maximum number of chars to search per field. |