From b4e3917d746fcf4be76d353be7c3b4ae9b229973 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 14 Dec 2018 11:28:18 +0100 Subject: Support for configuring and feeding float16 and bool field types --- .../com/yahoo/prelude/fastsearch/BoolField.java | 24 ++++++++++++++++ .../com/yahoo/prelude/fastsearch/ByteField.java | 3 -- .../yahoo/prelude/fastsearch/DocsumDefinition.java | 2 -- .../com/yahoo/prelude/fastsearch/DocsumField.java | 2 ++ .../com/yahoo/prelude/fastsearch/Float16Field.java | 33 ++++++++++++++++++++++ .../main/java/com/yahoo/prelude/query/Item.java | 3 +- .../java/com/yahoo/prelude/query/TermItem.java | 9 ++---- .../java/com/yahoo/prelude/query/WordItem.java | 19 ++++++------- .../main/java/com/yahoo/search/yql/YqlParser.java | 21 +++++++++----- .../com/yahoo/search/yql/YqlParserTestCase.java | 6 ++++ 10 files changed, 92 insertions(+), 30 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java (limited to 'container-search') diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java new file mode 100644 index 00000000000..633e76a2d9c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a byte field in the result set + * + */ + +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.search.result.NanNumber; + +/** + * @author bratseth + */ +public class BoolField extends DocsumField { + + public BoolField(String name) { + super(name); + } + + @Override + public Object convert(Inspector value) { return value.asBool(); } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java index 256edc66793..22069d0270c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java @@ -6,9 +6,6 @@ package com.yahoo.prelude.fastsearch; - -import java.nio.ByteBuffer; - import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 675c1af6545..cdf696ebe1e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -4,11 +4,9 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yahoo.data.access.Inspector; -import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 7d68e7b6679..777919286dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -48,10 +48,12 @@ public abstract class DocsumField { fieldFactory = new FieldFactory(); try { + fieldFactory.put("bool", BoolField.class); fieldFactory.put("byte", ByteField.class); fieldFactory.put("short", ShortField.class); fieldFactory.put("integer", IntegerField.class); fieldFactory.put("int64", Int64Field.class); + fieldFactory.put("float16", Float16Field.class); fieldFactory.put("float", FloatField.class); fieldFactory.put("double", DoubleField.class); fieldFactory.put("string", StringField.class); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java new file mode 100644 index 00000000000..f00d6b19d67 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Float16Field.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.search.result.NanNumber; + +/** + * A 16-bit float, represented as a (32-bit) Float in Java, as there is no 16-bit float support. + * + * @author bratseth + */ +public class Float16Field extends DocsumField { + + static final double EMPTY_VALUE = Float.NaN; + + public Float16Field(String name) { + super(name); + } + + private Object convert(float value) { + if (Float.isNaN(value)) { + return NanNumber.NaN; + } else { + return Float.valueOf(value); + } + } + + @Override + public Object convert(Inspector value) { + return convert((float)value.asDouble(EMPTY_VALUE)); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 0ba4133901a..d8b5fc9451a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -304,7 +304,7 @@ public abstract class Item implements Cloneable { public static void putString(String s, ByteBuffer buffer) { putBytes(Utf8.toBytes(s), buffer); } - public static void putBytes(byte [] bytes, ByteBuffer buffer) { + public static void putBytes(byte[] bytes, ByteBuffer buffer) { IntegerCompressor.putCompressedPositiveNumber(bytes.length, buffer); buffer.put(bytes); } @@ -324,6 +324,7 @@ public abstract class Item implements Cloneable { * TODO: Change the output query language into a canonical form of the input * query language */ + @Override public String toString() { StringBuilder buffer = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index e3edf3ea780..764fb2b1118 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -8,12 +8,8 @@ import java.nio.ByteBuffer; /** - *

A query term, that is, not only a term in the query language - * (an item), but also a term to be found in (or excluded from) - * the search index.

- * - *

Optionally, a TermItem may also specify the name of an - * index backend to search.

+ * Superclass of "leaf" conditions containing a single entity which is either matched in + * a field or not. * * @author bratseth * @author havardpe @@ -84,6 +80,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { */ public boolean isFromQuery() { return isFromQuery; } + @Override public abstract boolean isWords(); /** Sets the origin of this */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java index 246924ea9de..0b3d11158f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java @@ -9,14 +9,14 @@ import com.yahoo.protect.Validator; import java.nio.ByteBuffer; /** - * A term item containing a single word. + * A simple word or token to match in some field. * * @author bratseth * @author havardpe */ public class WordItem extends TermItem { - /** True if this is not part of the special tokens dictionary */ + /** True if this is not part of the special tokens dictionary */ private boolean words = true; /** Is this word stemmed? */ @@ -141,24 +141,20 @@ public class WordItem extends TermItem { } /** Word items uses a empty heading instead of "WORD " */ + @Override protected void appendHeadingString(StringBuilder buffer) {} + @Override public int hashCode() { return word.hashCode() + 71 * super.hashCode(); } + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } + if (!super.equals(object)) return false; WordItem other = (WordItem) object; // Ensured by superclass - - if (!this.word.equals(other.word)) { - return false; - } - - return true; + return this.word.equals(other.word); } public int getNumWords() { @@ -188,4 +184,5 @@ public class WordItem extends TermItem { discloser.addProperty("stemmed", stemmed); discloser.addProperty("words", words); } + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 6eaa1592a3e..5928dd23b28 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -52,6 +52,7 @@ import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.ToolBox; import com.yahoo.prelude.query.ToolBox.QueryVisitor; import com.yahoo.prelude.query.WandItem; @@ -942,13 +943,19 @@ public class YqlParser implements Parser { } @NonNull - private IntItem buildEquals(OperatorNode ast) { - IntItem number = new IntItem(fetchConditionWord(ast), fetchConditionIndex(ast)); - if (isIndexOnLeftHandSide(ast)) { - return leafStyleSettings(ast.getArgument(1, OperatorNode.class), number); - } else { - return leafStyleSettings(ast.getArgument(0, OperatorNode.class), number); - } + private TermItem buildEquals(OperatorNode ast) { + String value = fetchConditionWord(ast); + + TermItem item; + if (value.equals("true") || value.equals("false")) + item = new WordItem(value, fetchConditionIndex(ast)); + else + item = new IntItem(value, fetchConditionIndex(ast)); + + if (isIndexOnLeftHandSide(ast)) + return leafStyleSettings(ast.getArgument(1, OperatorNode.class), item); + else + return leafStyleSettings(ast.getArgument(0, OperatorNode.class), item); } @NonNull diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 8c47ae377c3..f2c53fb6b31 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -246,6 +246,12 @@ public class YqlParserTestCase { assertParse("select foo from bar where ([{\"filter\": true}]500) = price;", "|price:500"); } + @Test + public void testBoolean() { + assertParse("select foo from bar where flag = true;", "flag:true"); + assertParse("select foo from bar where flag = false;", "flag:false"); + } + @Test public void testTermAnnotations() { assertEquals("merkelapp", -- cgit v1.2.3