From 321ee8d0bffef15faf00474e3518009bab58f4cf Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 7 May 2018 15:15:55 +0200 Subject: Revert "Bratseth/lazy summary decoding" --- .../prelude/fastsearch/SlimeSummaryTestCase.java | 388 ++++++--------------- .../yahoo/prelude/fastsearch/partial-summary1.cfg | 10 - .../yahoo/prelude/fastsearch/partial-summary2.cfg | 12 - 3 files changed, 109 insertions(+), 301 deletions(-) delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg (limited to 'container-search/src/test') diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 83871b559b4..9b2de75e272 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -1,323 +1,167 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.google.common.collect.ImmutableSet; + import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.search.result.Hit; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; +import com.yahoo.slime.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import com.yahoo.slime.BinaryFormat; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import org.junit.Test; - +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -public class SlimeSummaryTestCase { - private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg"; - private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg"; +public class SlimeSummaryTestCase { @Test public void testDecodingEmpty() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul); + byte[] docsum = makeEmptyDocsum(); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertNull(hit.getField("integer_field")); - assertNull(hit.getField("short_field")); - assertNull(hit.getField("byte_field")); - assertNull(hit.getField("float_field")); - assertNull(hit.getField("double_field")); - assertNull(hit.getField("int64_field")); - assertNull(hit.getField("string_field")); - assertNull(hit.getField("data_field")); - assertNull(hit.getField("data_field")); - assertNull(hit.getField("longstring_field")); - assertNull(hit.getField("longdata_field")); - assertNull(hit.getField("longdata_field")); - assertNull(hit.getField("xmlstring_field")); - assertNull(hit.getField("xmlstring_field")); - assertNull(hit.getField("jsonstring_field")); + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("short_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("byte_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("float_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("double_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("int64_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("string_field"), equalTo("")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); + // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); + assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); + // Empty tensors are represented by null because we don't have type information here to create the right empty tensor assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } + private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); + } + @Test - public void testDecodingEmptyWithLegacyEmulation() { - LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); + public void testDecodingEmptyWithoutForcedFill() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false))); + byte[] docsum = makeEmptyDocsum(); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertEquals(NanNumber.NaN, hit.getField("integer_field")); - assertEquals(NanNumber.NaN, hit.getField("short_field")); - assertEquals(NanNumber.NaN, hit.getField("byte_field")); - assertEquals(NanNumber.NaN, hit.getField("float_field")); - assertEquals(NanNumber.NaN, hit.getField("double_field")); - assertEquals(NanNumber.NaN, hit.getField("int64_field")); - assertEquals("", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("", hit.getField("data_field").toString()); - assertEquals("", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("", hit.getField("xmlstring_field").toString()); - // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertEquals("", hit.getField("jsonstring_field").toString()); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(null)); + assertThat(hit.getField("short_field"), equalTo(null)); + assertThat(hit.getField("byte_field"), equalTo(null)); + assertThat(hit.getField("float_field"), equalTo(null)); + assertThat(hit.getField("double_field"), equalTo(null)); + assertThat(hit.getField("int64_field"), equalTo(null)); + assertThat(hit.getField("string_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("longstring_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("jsonstring_field"), equalTo(null)); assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } + private byte[] makeEmptyDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + return encode((slime)); + } + private byte [] encode(Slime slime) { + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + @Test public void testTimeout() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + byte[] docsum = makeTimeout(); FastHit hit = new FastHit(); - assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", - docsum.lazyDecode("default", timeoutSummary(), hit)); + assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", set.lazyDecode("default", docsum, hit)); } @Test public void testDecoding() { Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + byte[] docsum = makeDocsum(tensor1, tensor2); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit)); - assertEquals(4, hit.getField("integer_field")); - assertEquals((short)2, hit.getField("short_field")); - assertEquals((byte)1, hit.getField("byte_field")); - assertEquals(4.5F, hit.getField("float_field")); - assertEquals(8.75, hit.getField("double_field")); - assertEquals(8L, hit.getField("int64_field")); - assertEquals("string_value", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("data_value", hit.getField("data_field").toString()); - assertEquals("longstring_value", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("longdata_value", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("xmlstring_value", hit.getField("xmlstring_field").toString()); + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(4)); + assertThat(hit.getField("short_field"), equalTo((short)2)); + assertThat(hit.getField("byte_field"), equalTo((byte)1)); + assertThat(hit.getField("float_field"), equalTo(4.5f)); + assertThat(hit.getField("double_field"), equalTo(8.75)); + assertThat(hit.getField("int64_field"), equalTo(8L)); + assertThat(hit.getField("string_field"), equalTo("string_value")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("data_value")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("xmlstring_value")); if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); - assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); - assertNotNull(jstr.getParsedJSON()); + assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}")); + assertThat(jstr.getParsedJSON(), notNullValue()); - com.yahoo.data.access.Inspector value = jstr.inspect(); - assertEquals(1L, value.field("foo").asLong()); - assertEquals(2L, value.field("bar").asLong()); + com.yahoo.data.access.Inspectable obj = jstr; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); } else { StructuredData sdata = (StructuredData) hit.getField("jsonstring_field"); - assertEquals("{\"foo\":1,\"bar\":2}", sdata.toJson()); + assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}")); - com.yahoo.data.access.Inspector value = sdata.inspect(); - assertEquals(1L, value.field("foo").asLong()); - assertEquals(2L, value.field("bar").asLong()); + com.yahoo.data.access.Inspectable obj = sdata; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); } - @Test - public void testFieldAccessAPI() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); - DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); - DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); - FastHit hit = new FastHit(); - Map expected = new HashMap<>(); - - assertFields(expected, hit); - - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - assertFields(expected, hit); - - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - assertFields(expected, hit); - - hit.removeField("short_field"); - expected.remove("short_field"); - assertFields(expected, hit); - - hit.setField("string", "hello"); - expected.put("string", "hello"); - assertFields(expected, hit); - - hit.setField("short_field", 3.8F); - expected.put("short_field", 3.8F); - assertFields(expected, hit); - - hit.removeField("string"); - expected.remove("string"); - assertFields(expected, hit); - - hit.removeField("integer_field"); - hit.removeField("double_field"); - expected.remove("integer_field"); - expected.remove("double_field"); - assertFields(expected, hit); - - hit.clearFields(); - expected.clear(); - assertFields(expected, hit); - - // --- Re-populate - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - hit.setField("string1", "hello"); - hit.setField("string2", "hello"); - expected.put("string1", "hello"); - expected.put("string2", "hello"); - assertFields(expected, hit); - - Set keys = hit.fieldKeys(); - assertTrue(keys.remove("integer_field")); - expected.remove("integer_field"); - assertTrue(keys.remove("string2")); - expected.remove("string2"); - assertFields(expected, hit); - assertFalse(keys.remove("notpresent")); - - assertTrue(keys.retainAll(ImmutableSet.of("string1", "notpresent", "double_field"))); - expected.remove("short_field"); - expected.remove("float_field"); - assertFields(expected, hit); - - Iterator keyIterator = keys.iterator(); - assertEquals("string1", keyIterator.next()); - keyIterator.remove(); - expected.remove("string1"); - assertFields(expected, hit); - - assertEquals("double_field", keyIterator.next()); - keyIterator.remove(); - expected.remove("double_field"); - assertFields(expected, hit); - - // --- Re-populate - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - hit.setField("string", "hello"); - expected.put("string", "hello"); - assertFields(expected, hit); - - Iterator> fieldIterator = hit.fieldIterator(); - assertEquals("string", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("string"); - assertFields(expected, hit); - - fieldIterator.next(); - assertEquals("short_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("short_field"); - assertFields(expected, hit); - - fieldIterator.next(); - assertEquals("double_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("double_field"); - assertFields(expected, hit); - - fieldIterator = hit.fieldIterator(); - assertEquals("float_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("float_field"); - assertFields(expected, hit); - - assertEquals("integer_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("integer_field"); - assertFields(expected, hit); - } - - - /** Asserts that the expected fields are what is returned from every access method of Hit */ - private void assertFields(Map expected, Hit hit) { - // fieldKeys - int fieldNameIteratorFieldCount = 0; - for (Iterator i = hit.fieldKeys().iterator(); i.hasNext(); ) { - fieldNameIteratorFieldCount++; - assertTrue(expected.containsKey(i.next())); - } - assertEquals(expected.size(), fieldNameIteratorFieldCount); - assertEquals(expected.keySet(), hit.fieldKeys()); - // getField - for (Map.Entry field : expected.entrySet()) - assertEquals(field.getValue(), hit.getField(field.getKey())); - // fields - assertEquals(expected, hit.fields()); - // fieldIterator - int fieldIteratorFieldCount = 0; - for (Iterator> i = hit.fieldIterator(); i.hasNext(); ) { - fieldIteratorFieldCount++; - Map.Entry field = i.next(); - assertEquals(field.getValue(), expected.get(field.getKey())); - } - assertEquals(expected.size(), fieldIteratorFieldCount); - } - - private byte[] emptySummary() { - Slime slime = new Slime(); - slime.setObject(); - return encode((slime)); - } - - private byte [] timeoutSummary() { - Slime slime = new Slime(); - slime.setString("Timed out...."); - return encode((slime)); - } - - private byte[] partialSummary1() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setLong("integer_field", 4); - docsum.setLong("short_field", 2); - return encode((slime)); - } - - private byte[] partialSummary2() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setLong("integer_field", 4); - docsum.setDouble("float_field", 4.5); - docsum.setDouble("double_field", 8.75); - return encode((slime)); + private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); } - private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { + private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); docsum.setLong("integer_field", 4); @@ -341,24 +185,10 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte[] encode(Slime slime) { - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - - private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0)); - } - - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); + private byte [] makeTimeout() { + Slime slime = new Slime(); + slime.setString("Timed out...."); + return encode((slime)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg deleted file mode 100644 index 5aa5c84d936..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg +++ /dev/null @@ -1,10 +0,0 @@ -documentdb[1] -documentdb[0].name test -documentdb[0].summaryclass[1] -documentdb[0].summaryclass[0].name default -documentdb[0].summaryclass[0].id 1 -documentdb[0].summaryclass[0].fields[2] -documentdb[0].summaryclass[0].fields[0].name integer_field -documentdb[0].summaryclass[0].fields[0].type integer -documentdb[0].summaryclass[0].fields[1].name short_field -documentdb[0].summaryclass[0].fields[1].type short diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg deleted file mode 100644 index bc870a63d66..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg +++ /dev/null @@ -1,12 +0,0 @@ -documentdb[1] -documentdb[0].name test -documentdb[0].summaryclass[1] -documentdb[0].summaryclass[0].name default -documentdb[0].summaryclass[0].id 2 -documentdb[0].summaryclass[0].fields[3] -documentdb[0].summaryclass[0].fields[0].name integer_field -documentdb[0].summaryclass[0].fields[0].type integer -documentdb[0].summaryclass[0].fields[1].name float_field -documentdb[0].summaryclass[0].fields[1].type float -documentdb[0].summaryclass[0].fields[2].name double_field -documentdb[0].summaryclass[0].fields[2].type double -- cgit v1.2.3