diff options
Diffstat (limited to 'container-search/src/test/java/com/yahoo/prelude/fastsearch')
3 files changed, 301 insertions, 109 deletions
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 9b2de75e272..83871b559b4 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -1,167 +1,323 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; - +import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.result.Hit; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; -import com.yahoo.slime.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.*; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; - +import static org.junit.Assert.assertTrue; public class SlimeSummaryTestCase { + private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg"; + private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg"; + @Test public void testDecodingEmpty() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul); - byte[] docsum = makeEmptyDocsum(); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("short_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("byte_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("float_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("double_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("int64_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("string_field"), equalTo("")); - assertThat(hit.getField("data_field"), instanceOf(RawData.class)); - assertThat(hit.getField("data_field").toString(), equalTo("")); - assertThat(hit.getField("longstring_field"), equalTo((Object)"")); - assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); - assertThat(hit.getField("longdata_field").toString(), equalTo("")); - assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); - assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); - // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor + assertNull(docsum.lazyDecode("default", emptySummary(), hit)); + assertNull(hit.getField("integer_field")); + assertNull(hit.getField("short_field")); + assertNull(hit.getField("byte_field")); + assertNull(hit.getField("float_field")); + assertNull(hit.getField("double_field")); + assertNull(hit.getField("int64_field")); + assertNull(hit.getField("string_field")); + assertNull(hit.getField("data_field")); + assertNull(hit.getField("data_field")); + assertNull(hit.getField("longstring_field")); + assertNull(hit.getField("longdata_field")); + assertNull(hit.getField("longdata_field")); + assertNull(hit.getField("xmlstring_field")); + assertNull(hit.getField("xmlstring_field")); + assertNull(hit.getField("jsonstring_field")); assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); - } - @Test - public void testDecodingEmptyWithoutForcedFill() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false))); - byte[] docsum = makeEmptyDocsum(); + public void testDecodingEmptyWithLegacyEmulation() { + LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(null)); - assertThat(hit.getField("short_field"), equalTo(null)); - assertThat(hit.getField("byte_field"), equalTo(null)); - assertThat(hit.getField("float_field"), equalTo(null)); - assertThat(hit.getField("double_field"), equalTo(null)); - assertThat(hit.getField("int64_field"), equalTo(null)); - assertThat(hit.getField("string_field"), equalTo(null)); - assertThat(hit.getField("data_field"), equalTo(null)); - assertThat(hit.getField("data_field"), equalTo(null)); - assertThat(hit.getField("longstring_field"), equalTo(null)); - assertThat(hit.getField("longdata_field"), equalTo(null)); - assertThat(hit.getField("longdata_field"), equalTo(null)); - assertThat(hit.getField("xmlstring_field"), equalTo(null)); - assertThat(hit.getField("xmlstring_field"), equalTo(null)); - assertThat(hit.getField("jsonstring_field"), equalTo(null)); + assertNull(docsum.lazyDecode("default", emptySummary(), hit)); + assertEquals(NanNumber.NaN, hit.getField("integer_field")); + assertEquals(NanNumber.NaN, hit.getField("short_field")); + assertEquals(NanNumber.NaN, hit.getField("byte_field")); + assertEquals(NanNumber.NaN, hit.getField("float_field")); + assertEquals(NanNumber.NaN, hit.getField("double_field")); + assertEquals(NanNumber.NaN, hit.getField("int64_field")); + assertEquals("", hit.getField("string_field")); + assertEquals(RawData.class, hit.getField("data_field").getClass()); + assertEquals("", hit.getField("data_field").toString()); + assertEquals("", hit.getField("longstring_field")); + assertEquals(RawData.class, hit.getField("longdata_field").getClass()); + assertEquals("", hit.getField("longdata_field").toString()); + assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); + assertEquals("", hit.getField("xmlstring_field").toString()); + // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); + assertEquals("", hit.getField("jsonstring_field").toString()); + // Empty tensors are represented by null because we don't have type information here to create the right empty tensor assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } - private byte[] makeEmptyDocsum() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - return encode((slime)); - } - private byte [] encode(Slime slime) { - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - @Test public void testTimeout() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); - byte[] docsum = makeTimeout(); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", set.lazyDecode("default", docsum, hit)); + assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", + docsum.lazyDecode("default", timeoutSummary(), hit)); } @Test public void testDecoding() { Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); - - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); - byte[] docsum = makeDocsum(tensor1, tensor2); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(4)); - assertThat(hit.getField("short_field"), equalTo((short)2)); - assertThat(hit.getField("byte_field"), equalTo((byte)1)); - assertThat(hit.getField("float_field"), equalTo(4.5f)); - assertThat(hit.getField("double_field"), equalTo(8.75)); - assertThat(hit.getField("int64_field"), equalTo(8L)); - assertThat(hit.getField("string_field"), equalTo("string_value")); - assertThat(hit.getField("data_field"), instanceOf(RawData.class)); - assertThat(hit.getField("data_field").toString(), equalTo("data_value")); - assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); - assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); - assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value")); - assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); - assertThat(hit.getField("xmlstring_field").toString(), equalTo("<tag>xmlstring_value</tag>")); + assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit)); + assertEquals(4, hit.getField("integer_field")); + assertEquals((short)2, hit.getField("short_field")); + assertEquals((byte)1, hit.getField("byte_field")); + assertEquals(4.5F, hit.getField("float_field")); + assertEquals(8.75, hit.getField("double_field")); + assertEquals(8L, hit.getField("int64_field")); + assertEquals("string_value", hit.getField("string_field")); + assertEquals(RawData.class, hit.getField("data_field").getClass()); + assertEquals("data_value", hit.getField("data_field").toString()); + assertEquals("longstring_value", hit.getField("longstring_field")); + assertEquals(RawData.class, hit.getField("longdata_field").getClass()); + assertEquals("longdata_value", hit.getField("longdata_field").toString()); + assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); + assertEquals("<tag>xmlstring_value</tag>", hit.getField("xmlstring_field").toString()); if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); - assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}")); - assertThat(jstr.getParsedJSON(), notNullValue()); + assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); + assertNotNull(jstr.getParsedJSON()); - com.yahoo.data.access.Inspectable obj = jstr; - com.yahoo.data.access.Inspector value = obj.inspect(); - assertThat(value.field("foo").asLong(), equalTo(1L)); - assertThat(value.field("bar").asLong(), equalTo(2L)); + com.yahoo.data.access.Inspector value = jstr.inspect(); + assertEquals(1L, value.field("foo").asLong()); + assertEquals(2L, value.field("bar").asLong()); } else { StructuredData sdata = (StructuredData) hit.getField("jsonstring_field"); - assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}")); + assertEquals("{\"foo\":1,\"bar\":2}", sdata.toJson()); - com.yahoo.data.access.Inspectable obj = sdata; - com.yahoo.data.access.Inspector value = obj.inspect(); - assertThat(value.field("foo").asLong(), equalTo(1L)); - assertThat(value.field("bar").asLong(), equalTo(2L)); + com.yahoo.data.access.Inspector value = sdata.inspect(); + assertEquals(1L, value.field("foo").asLong()); + assertEquals(2L, value.field("bar").asLong()); } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0)); + @Test + public void testFieldAccessAPI() { + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); + DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); + FastHit hit = new FastHit(); + Map<String, Object> expected = new HashMap<>(); + + assertFields(expected, hit); + + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + assertFields(expected, hit); + + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + assertFields(expected, hit); + + hit.removeField("short_field"); + expected.remove("short_field"); + assertFields(expected, hit); + + hit.setField("string", "hello"); + expected.put("string", "hello"); + assertFields(expected, hit); + + hit.setField("short_field", 3.8F); + expected.put("short_field", 3.8F); + assertFields(expected, hit); + + hit.removeField("string"); + expected.remove("string"); + assertFields(expected, hit); + + hit.removeField("integer_field"); + hit.removeField("double_field"); + expected.remove("integer_field"); + expected.remove("double_field"); + assertFields(expected, hit); + + hit.clearFields(); + expected.clear(); + assertFields(expected, hit); + + // --- Re-populate + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + hit.setField("string1", "hello"); + hit.setField("string2", "hello"); + expected.put("string1", "hello"); + expected.put("string2", "hello"); + assertFields(expected, hit); + + Set<String> keys = hit.fieldKeys(); + assertTrue(keys.remove("integer_field")); + expected.remove("integer_field"); + assertTrue(keys.remove("string2")); + expected.remove("string2"); + assertFields(expected, hit); + assertFalse(keys.remove("notpresent")); + + assertTrue(keys.retainAll(ImmutableSet.of("string1", "notpresent", "double_field"))); + expected.remove("short_field"); + expected.remove("float_field"); + assertFields(expected, hit); + + Iterator<String> keyIterator = keys.iterator(); + assertEquals("string1", keyIterator.next()); + keyIterator.remove(); + expected.remove("string1"); + assertFields(expected, hit); + + assertEquals("double_field", keyIterator.next()); + keyIterator.remove(); + expected.remove("double_field"); + assertFields(expected, hit); + + // --- Re-populate + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + hit.setField("string", "hello"); + expected.put("string", "hello"); + assertFields(expected, hit); + + Iterator<Map.Entry<String, Object>> fieldIterator = hit.fieldIterator(); + assertEquals("string", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("string"); + assertFields(expected, hit); + + fieldIterator.next(); + assertEquals("short_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("short_field"); + assertFields(expected, hit); + + fieldIterator.next(); + assertEquals("double_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("double_field"); + assertFields(expected, hit); + + fieldIterator = hit.fieldIterator(); + assertEquals("float_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("float_field"); + assertFields(expected, hit); + + assertEquals("integer_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("integer_field"); + assertFields(expected, hit); } - private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) { + + /** Asserts that the expected fields are what is returned from every access method of Hit */ + private void assertFields(Map<String, Object> expected, Hit hit) { + // fieldKeys + int fieldNameIteratorFieldCount = 0; + for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) { + fieldNameIteratorFieldCount++; + assertTrue(expected.containsKey(i.next())); + } + assertEquals(expected.size(), fieldNameIteratorFieldCount); + assertEquals(expected.keySet(), hit.fieldKeys()); + // getField + for (Map.Entry<String, Object> field : expected.entrySet()) + assertEquals(field.getValue(), hit.getField(field.getKey())); + // fields + assertEquals(expected, hit.fields()); + // fieldIterator + int fieldIteratorFieldCount = 0; + for (Iterator<Map.Entry<String, Object>> i = hit.fieldIterator(); i.hasNext(); ) { + fieldIteratorFieldCount++; + Map.Entry<String, Object> field = i.next(); + assertEquals(field.getValue(), expected.get(field.getKey())); + } + assertEquals(expected.size(), fieldIteratorFieldCount); + } + + private byte[] emptySummary() { + Slime slime = new Slime(); + slime.setObject(); + return encode((slime)); + } + + private byte [] timeoutSummary() { + Slime slime = new Slime(); + slime.setString("Timed out...."); + return encode((slime)); + } + + private byte[] partialSummary1() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + return encode((slime)); + } + + private byte[] partialSummary2() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + return encode((slime)); + } + + private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); docsum.setLong("integer_field", 4); @@ -185,10 +341,24 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte [] makeTimeout() { - Slime slime = new Slime(); - slime.setString("Timed out...."); - return encode((slime)); + private byte[] encode(Slime slime) { + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg new file mode 100644 index 00000000000..5aa5c84d936 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg @@ -0,0 +1,10 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 1 +documentdb[0].summaryclass[0].fields[2] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name short_field +documentdb[0].summaryclass[0].fields[1].type short diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg new file mode 100644 index 00000000000..bc870a63d66 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg @@ -0,0 +1,12 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 2 +documentdb[0].summaryclass[0].fields[3] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name float_field +documentdb[0].summaryclass[0].fields[1].type float +documentdb[0].summaryclass[0].fields[2].name double_field +documentdb[0].summaryclass[0].fields[2].type double |