diff options
Diffstat (limited to 'container-search')
18 files changed, 213 insertions, 133 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java index 0e54adae932..7046cb91cd3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java @@ -16,11 +16,11 @@ import com.yahoo.prelude.hitfield.RawData; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.Inspector; - /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class DataField extends DocsumField implements VariableLengthField { + public DataField(String name) { super(name); } @@ -67,4 +67,5 @@ public class DataField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return convert(value.asData(Value.empty().asData())); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 2f0768d4e8b..3b414e288da 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -23,6 +23,7 @@ import java.util.logging.Logger; * @author Bjørn Borud */ public final class DocsumDefinitionSet { + public static final int SLIME_MAGIC_ID = 0x55555555; private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); @@ -73,6 +74,7 @@ public final class DocsumDefinitionSet { buffer.order(ByteOrder.LITTLE_ENDIAN); long docsumClassId = buffer.getInt(); if (docsumClassId != SLIME_MAGIC_ID) { + // TODO: Not used, remove - bratseth 2017-01-016 DocsumDefinition docsumDefinition = lookupDocsum(docsumClassId); Docsum docsum = new Docsum(docsumDefinition, data); hit.addSummary(docsum); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 3aa02f57a1e..1e14fea26b7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -13,8 +13,8 @@ import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.log.LogLevel; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Bjørn Borud + * @author Steinar Knutsen */ public abstract class DocsumField { @@ -22,26 +22,26 @@ public abstract class DocsumField { private static FieldFactory fieldFactory; private static class FieldFactory { + Map<String, Constructor<? extends DocsumField>> constructors = new HashMap<>(); - void put(final String typename, - final Class<? extends DocsumField> fieldClass) - throws NoSuchMethodException, SecurityException - { - final Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class); + void put(String typename, Class<? extends DocsumField> fieldClass) + throws NoSuchMethodException, SecurityException { + Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class); constructors.put(typename, constructor); } - DocsumField create(final String typename, final String name, final LegacyEmulationConfig emulConfig) + DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig) throws InstantiationException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException - { + IllegalArgumentException, InvocationTargetException { DocsumField f = constructors.get(typename).newInstance(name); f.emulConfig = emulConfig; return f; } } + private LegacyEmulationConfig emulConfig; + final LegacyEmulationConfig getEmulConfig() { return emulConfig; } static { @@ -61,27 +61,27 @@ public abstract class DocsumField { fieldFactory.put("jsonstring", StructDataField.class); fieldFactory.put("featuredata", FeatureDataField.class); fieldFactory.put("xmlstring", XMLField.class); - } catch (final Exception e) { - log.log(LogLevel.ERROR, - "Could not initialize docsum decoding properly.", e); + fieldFactory.put("tensor", TensorField.class); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e); } } protected String name; - protected DocsumField(final String name) { + protected DocsumField(String name) { this.name = name; } /* for unit test only */ - static DocsumField create(final String name, final String typename) { + static DocsumField create(String name, String typename) { return create(name, typename, new LegacyEmulationConfig()); } - public static DocsumField create(final String name, final String typename, LegacyEmulationConfig emulConfig) { + public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) { try { return fieldFactory.create(typename, name, emulConfig); - } catch (final Exception e) { + } catch (Exception e) { throw new RuntimeException("Unknown field type '" + typename + "'", e); } } @@ -90,7 +90,7 @@ public abstract class DocsumField { return name; } - public boolean isCompressed(final ByteBuffer b) { + public boolean isCompressed(ByteBuffer b) { return false; } @@ -106,8 +106,8 @@ public abstract class DocsumField { public abstract Object decode(ByteBuffer b); /** - * Get the number of bytes this field occupies in the given buffer and set - * the position of the first byte after this field. + * Get the number of bytes this field occupies in the given buffer + * AND SET(!) the position to the first byte after this field. */ public abstract int getLength(ByteBuffer b); @@ -116,4 +116,5 @@ public abstract class DocsumField { * for this field. **/ public abstract Object convert(Inspector value); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index fe0c4a35d1e..91e67174c4c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -174,8 +174,7 @@ public class FastHit extends Hit { } /** - * - * @param useRowInIndexUri Sets whether to use the row in the index uri. See FastSearcher for details. + * Sets whether to use the row in the index uri. See FastSearcher for details. */ public void setUseRowInIndexUri(boolean useRowInIndexUri) { this.useRowInIndexUri = useRowInIndexUri; @@ -189,7 +188,7 @@ public class FastHit extends Hit { } /** - * @return the row number where this hit originated, or 0 if not known + * Returns the row number where this hit originated, or 0 if not known * */ public int getRow() { if (rowBits == 0) { @@ -214,17 +213,20 @@ public class FastHit extends Hit { * filled returns the following types, even when the field has no actual value:</p> * * <ul> - * <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li> - * <li><b>string/uri/content</b>: A Java String.<br> + * <li><b>string and uri fields</b>: A Java String.<br> * The empty string ("") if no value is assigned in the document. * + * <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li> + * * <li><b>Numerics</b>: The corresponding numeric Java type.<br> * If the field has <i>no value</i> assigned in the document, * the special numeric {@link com.yahoo.search.result.NanNumber#NaN} is returned. * - * <li><b>raw</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance + * <li><b>raw fields</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance + * + * <li><b>tensor fields</b>: A {@link com.yahoo.tensor.Tensor} instance * - * <li><b>multivalue fields</b>: A {@link com.yahoo.prelude.hitfield.JSONString} instance + * <li><b>multivalue fields</b>: A {@link com.yahoo.data.access.Inspector} instance * </ul> */ @Override @@ -277,10 +279,7 @@ public class FastHit extends Hit { for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); if (value.type() == Type.STRING && - (field instanceof LongstringField || - field instanceof StringField || - field instanceof XMLField)) - { + (field instanceof LongstringField || field instanceof StringField || field instanceof XMLField)) { setDocsumFieldIfNotPresent(fieldName, new LazyString(field, value)); } else { Inspector f = value.field(fieldName); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java index d61a15723ac..87ee906e592 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; - import java.nio.ByteBuffer; import com.yahoo.io.SlowInflate; @@ -10,13 +9,13 @@ import com.yahoo.text.Utf8; import com.yahoo.data.access.*; import com.yahoo.data.access.simple.Value; - /** - * Class representing a JSON string field in the result set + * A hit field containing JSON structured data * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JSONField extends DocsumField implements VariableLengthField { + public JSONField(String name) { super(name); } @@ -104,9 +103,7 @@ public class JSONField extends DocsumField implements VariableLengthField { } } - private static class ArrConv extends CompatibilityConverter - implements ArrayTraverser - { + private static class ArrConv extends CompatibilityConverter implements ArrayTraverser { @Override public void entry(int idx, Inspector value) { target.add(stringify(value)); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java index 617f382f462..3ad8ba510aa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -5,7 +5,6 @@ */ package com.yahoo.prelude.fastsearch; - import java.nio.ByteBuffer; import com.yahoo.io.SlowInflate; @@ -13,11 +12,11 @@ import com.yahoo.prelude.hitfield.RawData; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.Inspector; - /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class LongdataField extends DocsumField implements VariableLengthField { + public LongdataField(String name) { super(name); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java index 744476beaa5..2cb3ac51873 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java @@ -5,18 +5,17 @@ */ package com.yahoo.prelude.fastsearch; - import java.nio.ByteBuffer; import com.yahoo.io.SlowInflate; import com.yahoo.text.Utf8; import com.yahoo.data.access.Inspector; - /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class LongstringField extends DocsumField implements VariableLengthField { + public LongstringField(String name) { super(name); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java index e9c19590102..d30ab3e87e2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java @@ -12,9 +12,8 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ - public class ShortField extends DocsumField { static final short EMPTY_VALUE = Short.MIN_VALUE; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java index 671188e4cae..f5be5ffb678 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java @@ -13,9 +13,10 @@ import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class StringField extends DocsumField implements VariableLengthField { + public StringField(String name) { super(name); } @@ -59,4 +60,5 @@ public class StringField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return value.asString(""); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java index f0f4b82c22a..c83f209e8f0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java @@ -8,7 +8,7 @@ import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.JSONString; /** - * Class representing a XML rendered structured data field in the result set + * A hit field containing JSON structured data */ public class StructDataField extends JSONField { @@ -22,9 +22,7 @@ public class StructDataField extends JSONField { } public Object convert(Inspector value) { - if (getEmulConfig().stringBackedStructuredData() || - value.type() == Type.STRING) - { + if (getEmulConfig().stringBackedStructuredData() || value.type() == Type.STRING) { return super.convert(value); } return new StructuredData(value); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java new file mode 100644 index 00000000000..0d97a5bdaf7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java @@ -0,0 +1,66 @@ +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.simple.Value; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; + +import java.nio.ByteBuffer; +import java.util.Optional; + +/** + * A tensor field. Tensors are encoded as a data field where the data (following the length) + * is encoded in a tensor binary format defined by com.yahoo.tensor.serialization.TypedBinaryFormat + * + * @author bratseth + */ +public class TensorField extends DocsumField implements VariableLengthField { + + public TensorField(String name) { + super(name); + } + + @Override + public Tensor decode(ByteBuffer buffer) { + int length = buffer.getInt(); + if (length == 0) return null; + ByteBuffer contentBuffer = ByteBuffer.wrap(buffer.array(), buffer.arrayOffset() + buffer.position(), length); + Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), new GrowableByteBuffer(contentBuffer)); + buffer.position(buffer.position() + length); + return tensor; + } + + @Override + public Tensor decode(ByteBuffer b, FastHit hit) { + Tensor tensor = decode(b); + hit.setField(name, tensor); + return tensor; + } + + @Override + public String toString() { + return "field " + getName() + " type tensor"; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + int length = b.getInt(); + b.position(offset + length); + return length; + } + + @Override + public int sizeOfLength() { + return 4; + } + + @Override + public Object convert(Inspector value) { + byte[] content = value.asData(Value.empty().asData()); + if (content.length == 0) return null; + return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(content)); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java index f169533f8db..2ff7c4864e2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java @@ -5,8 +5,11 @@ package com.yahoo.prelude.fastsearch; * Interface to easier find the start of the actual data for variable length * fields. * - * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface VariableLengthField { - public int sizeOfLength(); + + /** Returns the size of the length preceeding the data of this field, in bytes */ + int sizeOfLength(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java index 26787e442fc..d9b1a9a57ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.hitfield; /** - * A representation of some random data with unknown semantics + * A representation of some binary data with unknown semantics * * @author arnej27959 */ diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java index d8321a579b3..bd33e3bcd63 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java @@ -134,7 +134,7 @@ import java.util.logging.Logger; * The content of any additional response header fields. * </dl> * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public abstract class HTTPSearcher extends ClusterSearcher<Connection> { diff --git a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java index 6e750ea0102..34bfefbe98b 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java @@ -17,9 +17,10 @@ import static junit.framework.TestCase.*; /** * Tests the PacketDecoder * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjørn Borud */ public class PacketDecoderTestCase { + static byte[] queryResultPacketData = new byte[] {0,0,0,104, 0,0,0,217-256, @@ -44,7 +45,6 @@ public class PacketDecoderTestCase { data.put(queryResultPacketData); data.flip(); - // not really necessary for testing, but these help visualize // the state the buffer should be in so a reader of this test // will not have to diff --git a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java index 5cf9aeda978..64153363ca5 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java @@ -109,7 +109,7 @@ public class RankFeaturesTestCase { if (key.contains(".type")) { result.put(key, Utf8.toString(value)); } else { - result.put(key, TypedBinaryFormat.decode(type, value)); + result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value))); } } return result; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 47a3003371e..f45d1e0fd2e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -4,81 +4,28 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.search.LegacyEmulationConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.Docsum; -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; -import com.yahoo.document.DocumentId; -import com.yahoo.document.GlobalId; import com.yahoo.slime.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class SlimeSummaryTestCase { - public static DocsumDefinitionSet createDocsumDefinitionSet(String configID) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0)); - } - - public static DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); - } - - public byte[] makeEmptyDocsum() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - - public byte[] makeDocsum() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setLong("integer_field", 4); - docsum.setLong("short_field", 2); - docsum.setLong("byte_field", 1); - docsum.setDouble("float_field", 4.5); - docsum.setDouble("double_field", 8.75); - docsum.setLong("int64_field", 8); - docsum.setString("string_field", "string_value"); - docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); - docsum.setString("longstring_field", "longstring_value"); - docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); - docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>"); - { - Cursor field = docsum.setObject("jsonstring_field"); - field.setLong("foo", 1); - field.setLong("bar", 2); - } - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - @Test public void testDecodingEmpty() { String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; @@ -87,13 +34,13 @@ public class SlimeSummaryTestCase { byte[] docsum = makeEmptyDocsum(); FastHit hit = new FastHit(); set.lazyDecode("default", docsum, hit); - assertThat(hit.getField("integer_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("short_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("byte_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("float_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("double_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("int64_field"), equalTo((Object) NanNumber.NaN)); - assertThat(hit.getField("string_field"), equalTo((Object)"")); + assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("short_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("byte_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("float_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("double_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("int64_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("string_field"), equalTo("")); assertThat(hit.getField("data_field"), instanceOf(RawData.class)); assertThat(hit.getField("data_field").toString(), equalTo("")); assertThat(hit.getField("longstring_field"), equalTo((Object)"")); @@ -103,6 +50,14 @@ public class SlimeSummaryTestCase { assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); + // Empty tensors are represented by null because we don't have type information here to create the right empty tensor + assertNull(hit.getField("tensor_field1")); + assertNull(hit.getField("tensor_field2")); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); } @Test @@ -127,22 +82,39 @@ public class SlimeSummaryTestCase { assertThat(hit.getField("xmlstring_field"), equalTo(null)); assertThat(hit.getField("xmlstring_field"), equalTo(null)); assertThat(hit.getField("jsonstring_field"), equalTo(null)); + assertNull(hit.getField("tensor_field1")); + assertNull(hit.getField("tensor_field2")); + } + + private byte[] makeEmptyDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); } @Test public void testDecoding() { + Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); + Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); - byte[] docsum = makeDocsum(); + byte[] docsum = makeDocsum(tensor1, tensor2); FastHit hit = new FastHit(); set.lazyDecode("default", docsum, hit); - assertThat(hit.getField("integer_field"), equalTo((Object)new Integer(4))); - assertThat(hit.getField("short_field"), equalTo((Object)new Short((short)2))); - assertThat(hit.getField("byte_field"), equalTo((Object)new Byte((byte)1))); - assertThat(hit.getField("float_field"), equalTo((Object)new Float(4.5f))); - assertThat(hit.getField("double_field"), equalTo((Object)new Double(8.75))); - assertThat(hit.getField("int64_field"), equalTo((Object)new Long(8L))); - assertThat(hit.getField("string_field"), equalTo((Object)"string_value")); + assertThat(hit.getField("integer_field"), equalTo(4)); + assertThat(hit.getField("short_field"), equalTo((short)2)); + assertThat(hit.getField("byte_field"), equalTo((byte)1)); + assertThat(hit.getField("float_field"), equalTo(4.5f)); + assertThat(hit.getField("double_field"), equalTo(8.75)); + assertThat(hit.getField("int64_field"), equalTo(8L)); + assertThat(hit.getField("string_field"), equalTo("string_value")); assertThat(hit.getField("data_field"), instanceOf(RawData.class)); assertThat(hit.getField("data_field").toString(), equalTo("data_value")); assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); @@ -168,5 +140,43 @@ public class SlimeSummaryTestCase { assertThat(value.field("foo").asLong(), equalTo(1L)); assertThat(value.field("bar").asLong(), equalTo(2L)); } + assertEquals(tensor1, hit.getField("tensor_field1")); + assertEquals(tensor2, hit.getField("tensor_field2")); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); + } + + private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + docsum.setLong("byte_field", 1); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + docsum.setLong("int64_field", 8); + docsum.setString("string_field", "string_value"); + docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("longstring_field", "longstring_value"); + docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>"); + { + Cursor field = docsum.setObject("jsonstring_field"); + field.setLong("foo", 1); + field.setLong("bar", 2); + } + docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); + docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg index a188754db19..e46904b17d0 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg @@ -3,7 +3,7 @@ documentdb[0].name test documentdb[0].summaryclass[1] documentdb[0].summaryclass[0].name default documentdb[0].summaryclass[0].id 0 -documentdb[0].summaryclass[0].fields[12] +documentdb[0].summaryclass[0].fields[14] documentdb[0].summaryclass[0].fields[0].name integer_field documentdb[0].summaryclass[0].fields[0].type integer documentdb[0].summaryclass[0].fields[1].name short_field @@ -28,3 +28,7 @@ documentdb[0].summaryclass[0].fields[10].name xmlstring_field documentdb[0].summaryclass[0].fields[10].type xmlstring documentdb[0].summaryclass[0].fields[11].name jsonstring_field documentdb[0].summaryclass[0].fields[11].type jsonstring +documentdb[0].summaryclass[0].fields[12].name tensor_field1 +documentdb[0].summaryclass[0].fields[12].type tensor +documentdb[0].summaryclass[0].fields[13].name tensor_field2 +documentdb[0].summaryclass[0].fields[13].type tensor |