From a6e03072aea85d3595755d4dc3fb885d60ba85d6 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 8 May 2018 16:02:08 +0200 Subject: Revert "Revert "Bratseth/lazy summary decoding"" --- .../yahoo/prelude/fastsearch/DocsumDefinition.java | 13 +- .../java/com/yahoo/prelude/fastsearch/FastHit.java | 584 ++++++++++++++++++++- .../src/main/java/com/yahoo/search/Query.java | 12 +- .../src/main/java/com/yahoo/search/result/Hit.java | 33 +- .../com/yahoo/search/yql/MinimalQueryInserter.java | 4 +- .../prelude/fastsearch/SlimeSummaryTestCase.java | 388 ++++++++++---- .../yahoo/prelude/fastsearch/partial-summary1.cfg | 10 + .../yahoo/prelude/fastsearch/partial-summary2.cfg | 12 + 8 files changed, 907 insertions(+), 149 deletions(-) create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index d50006fb82c..e20340a03c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -11,6 +11,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * A docsum definition which knows how to decode a certain class of document @@ -66,9 +67,15 @@ public class DocsumDefinition { return fields.get(fieldIndex); } - /** Returns the index of a field name */ - public Integer getFieldIndex(String fieldName) { - return fieldNameToIndex.get(fieldName); + /** Returns the field with this name, or null if none */ + public DocsumField getField(String fieldName) { + Integer index = fieldNameToIndex.get(fieldName); + if (index == null) return null; + return getField(index); + } + + public Set fieldNames() { + return fieldNameToIndex.keySet(); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index f1f5e2b7403..298bd839393 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.yahoo.data.access.ObjectTraverser; import com.yahoo.document.GlobalId; import com.yahoo.fs4.QueryPacketData; import com.yahoo.net.URI; @@ -8,6 +9,17 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.Relevance; import com.yahoo.data.access.Inspector; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + /** * A regular hit from a Vespa backend * @@ -33,6 +45,24 @@ public class FastHit extends Hit { private transient QueryPacketData queryPacketData = null; private transient CacheKey cacheKey = null; + /** + * Summaries added to this hit which are not yet decoded into fields. + * Fields are resolved by returning the first non-null value found by + * 1) the field value from the Map of fields in the Hit supertype, and + * 2) each of the summaries, in the order of the list (which is the add order). + * This ensures that values set from code overwrites any value received as + * summary data. + * + * The reason we keep this rather than eagerly decoding into a the field map + * is to reduce garbage collection and decoding cost, with the assumption + * that most fields passes through the container with no processing most + * of the time. + */ + private List summaries = new ArrayList<>(1); + + /** Removed field values, which should therefore not be returned if present in summary data */ + private Set removedFields = null; + /** * Creates an empty and temporarily invalid summary hit */ @@ -128,16 +158,9 @@ public class FastHit extends Hit { /** For internal use */ public void addSummary(DocsumDefinition docsumDef, Inspector value) { - reserve(docsumDef.getFieldCount()); - for (DocsumField field : docsumDef.getFields()) { - String fieldName = field.getName(); - Inspector f = value.field(fieldName); - if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) { - if (super.getField(fieldName) == null) { - setField(fieldName, field.convert(f)); - } - } - } + if (removedFields != null) + removedFields.removeAll(docsumDef.fieldNames()); + summaries.add(new SummaryData(this, docsumDef, value, summaries.size())); } /** @@ -172,8 +195,132 @@ public class FastHit extends Hit { * */ @Override - public Object getField(String key) { - return super.getField(key); + public Object getField(String name) { + Object value = super.getField(name); + if (value != null) return value; + value = getSummaryValue(name); + if (value != null) + super.setField(name, value); + return value; + } + + @Override + public Object setField(String name, Object value) { + if (removedFields != null) { + if (removedFields.remove(name)) { + if (removedFields.isEmpty()) + removedFields = null; + } + } + Object oldValue = super.setField(name, value); + if (oldValue != null) return oldValue; + return getSummaryValue(name); + } + + /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ + @Override + public Map fields() { + Map fields = new HashMap<>(); + for (Iterator> i = fieldIterator(); i.hasNext(); ) { + Map.Entry field = i.next(); + fields.put(field.getKey(), field.getValue()); + } + return fields; + } + + /** Returns a modifiable iterator over the fields of this */ + @Override + public Iterator> fieldIterator() { + return new FieldIterator(this, super.fieldIterator()); + } + + /** Returns a modifiable iterator over the field names of this */ + Iterator fieldNameIterator() { + return new FieldNameIterator(this, super.fieldKeys().iterator()); + } + + /** Removes all fields of this */ + @Override + public void clearFields() { + summaries.clear(); + if (removedFields != null) + removedFields = null; + super.clearFields(); + } + + /** + * Removes a field from this + * + * @return the removed value of the field, or null if none + */ + @Override + public Object removeField(String name) { + Object removedValue = super.removeField(name); + if (removedValue == null) + removedValue = getSummaryValue(name); + + if (removedValue != null) { + if (removedFields == null) + removedFields = new HashSet<>(2); + removedFields.add(name); + } + + return removedValue; + } + + /** + * Returns the keys of the fields of this hit as a modifiable view. + * This follows the rules of key sets returned from maps: Key removals are reflected + * in the map, add and addAll is not supported. + */ + @Override + public Set fieldKeys() { + return new FieldSet(this); + } + + private Set mapFieldKeys() { + return super.fieldKeys(); + } + + /** Returns whether this field is present in the field map in the parent hit */ + // Note: If this is made public it must be changed to also check the summary data + // (and internal usage must change to another method). + @Override + protected boolean hasField(String name) { + return super.hasField(name); + } + + /** Returns whether any fields are present in the field map in the parent hit */ + // Note: If this is made public it must be changed to also check the summary data + // (and internal usage must change to another method). + @Override + protected boolean hasFields() { + return super.hasFields(); + } + + /** + * Changes the key under which a value is found. This is useful because it allows keys to be changed + * without accessing the value (which may be lazily created). + * + * @deprecated do not use + */ + @Deprecated + @Override + @SuppressWarnings("deprecation") + public void changeFieldKey(String oldKey, String newKey) { + Object value = removeField(oldKey); + if (value != null) + setField(newKey, value); + } + + private Object getSummaryValue(String name) { + if (removedFields != null && removedFields.contains(name)) + return null; + for (SummaryData summaryData : summaries) { + Object value = summaryData.getField(name); + if (value != null) return value; + } + return null; } @Override @@ -192,6 +339,8 @@ public class FastHit extends Hit { } } + /** @deprecated do not use */ + @Deprecated // TODO: Make private on Vespa 7 public static String asHexString(GlobalId gid) { StringBuilder sb = new StringBuilder(); byte[] rawGid = gid.getRawId(); @@ -205,4 +354,415 @@ public class FastHit extends Hit { return sb.toString(); } + /** A set view of all the field names in this hit. Add/addAll is not supported but remove is. */ + private static class FieldSet implements Set { + + // A set implementation which tries to avoid creating an actual set when possible. + // With more work this could be optimized to avoid creating the set in additional cases. + + private final FastHit hit; + + /** The computed set of fields. Lazily created as it is not always needed. */ + private Set fieldSet = null; + + public FieldSet(FastHit hit) { + this.hit = hit; + } + + @Override + public int size() { + return createSet().size(); + } + + @Override + public boolean isEmpty() { + if ( ! hit.hasFields() && hit.summaries.isEmpty()) return true; + return createSet().isEmpty(); + } + + @Override + public boolean contains(Object o) { + String field = (String)o; + if (hit.hasField(field)) return true; + return createSet().contains(field); + } + + @Override + public Object[] toArray() { + return createSet().toArray(); + } + + @Override + public T[] toArray(T[] a) { + return createSet().toArray(a); + } + + @Override + public boolean add(String s) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object o) { + String field = (String)o; + boolean removed = hit.removeField(field) != null; + if (fieldSet != null) + fieldSet.remove(field); + return removed; + } + + @Override + public boolean containsAll(Collection c) { + for (Object field : c) + if ( ! contains(field)) + return false; + return true; + } + + @Override + public boolean addAll(Collection c) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean retainAll(Collection c) { + Set toRetain = c instanceof Set ? (Set)c : new HashSet(c); + boolean anyRemoved = false; + for (Iterator i = iterator(); i.hasNext(); ) { + String field = i.next(); + if (toRetain.contains(field)) continue; + + i.remove(); + anyRemoved = true; + } + return anyRemoved; + } + + @Override + public boolean removeAll(Collection c) { + boolean anyRemoved = false; + for (Object field : c) + if (remove(field)) + anyRemoved = true; + return anyRemoved; + } + + @Override + public void clear() { + hit.clearFields(); + fieldSet = null; + } + + @Override + public Iterator iterator() { + return hit.fieldNameIterator(); + } + + @Override + public int hashCode() { + return createSet().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Set)) return false; + return createSet().equals(o); + } + + @Override + public String toString() { + return createSet().toString(); + } + + private Set createSet() { + if (this.fieldSet != null) return this.fieldSet; + if ( ! hit.hasFields() && hit.summaries.isEmpty()) return Collections.emptySet(); // shortcut + + Set fields = new HashSet<>(); + if (hit.hasFields()) + fields.addAll(hit.mapFieldKeys()); + + for (SummaryData summaryData : hit.summaries) + summaryData.data.traverse((ObjectTraverser)(name, __) -> fields.add(name)); + if (hit.removedFields != null) + fields.removeAll(hit.removedFields); + + this.fieldSet = fields; + + return fields; + } + + } + + /** Summary data (values of a number of fields) received for this hit */ + private static class SummaryData { + + private final FastHit hit; + private final DocsumDefinition type; + private final Inspector data; + + /** The index of this summary in the list of summaries added to this */ + private final int index; + + SummaryData(FastHit hit, DocsumDefinition type, Inspector data, int index) { + this.hit = hit; + this.type = type; + this.data = data; + this.index = index; + } + + Object getField(String name) { + DocsumField fieldType = type.getField(name); + if (fieldType == null) return null; + Inspector fieldValue = data.field(name); + if ( ! fieldValue.valid() && ! fieldType.getEmulConfig().forceFillEmptyFields()) return null; + return fieldType.convert(fieldValue); + } + + /** Decodes the given summary into the field map in the parent */ + private void decodeInto(FastHit hit) { + hit.reserve(type.getFieldCount()); + for (DocsumField field : type.getFields()) { + String fieldName = field.getName(); + Inspector f = data.field(fieldName); + if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) { + if (hit.getField(fieldName) == null) + hit.setField(fieldName, field.convert(f)); + } + } + } + + private Iterator> fieldIterator() { + return new SummaryDataFieldIterator(hit, type, data.fields().iterator(), index); + } + + private Iterator fieldNameIterator() { + return new SummaryDataFieldNameIterator(hit, data.fields().iterator(), index); + } + + /** + * Abstract superclass of iterators over SummaryData content which takes care of skipping unknown, + * removed and already returned fields. Read only. + */ + private static abstract class SummaryDataIterator implements Iterator { + + private final FastHit hit; + private final Iterator> fieldIterator; + private final int index; + + /** The next value or null if none, eagerly read because we need to skip removed and overwritten values */ + private VALUE next; + + SummaryDataIterator(FastHit hit, + Iterator> fieldIterator, + int index) { + this.hit = hit; + this.fieldIterator = fieldIterator; + this.index = index; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public VALUE next() { + if (next == null) throw new NoSuchElementException(); + + VALUE returnValue = next; + advanceNext(); + return returnValue; + } + + protected abstract VALUE toValue(Map.Entry field); + + protected void advanceNext() { + while (fieldIterator.hasNext()) { + Map.Entry nextEntry = fieldIterator.next(); + String fieldName = nextEntry.getKey(); + next = toValue(nextEntry); + if ( next != null && + ! hit.hasField(fieldName) && + ! isRemoved(fieldName) && + ! isAlreadyReturned(fieldName)) + return; + } + next = null; + } + + private boolean isRemoved(String fieldName) { + return hit.removedFields != null && hit.removedFields.contains(fieldName); + } + + private boolean isAlreadyReturned(String fieldName) { + for (int i = 0; i < index; i++) { + if (hit.summaries.get(i).type.fieldNames().contains(fieldName)) + return true; + } + return false; + } + + } + + + /** Iterator over the fields in a SummaryData instance. Read only. */ + private static class SummaryDataFieldIterator extends SummaryDataIterator> { + + private final DocsumDefinition type; + + SummaryDataFieldIterator(FastHit hit, + DocsumDefinition type, + Iterator> fieldIterator, + int index) { + super(hit, fieldIterator, index); + this.type = type; + advanceNext(); + } + + @Override + protected Map.Entry toValue(Map.Entry field) { + DocsumField fieldType = type.getField(field.getKey()); + if (fieldType == null) return null; // type and content mismatch: May happen during reconfig + return new SummaryFieldEntry(field.getKey(), fieldType.convert(field.getValue())); + } + + private static final class SummaryFieldEntry implements Map.Entry { + + private final String key; + private final Object value; + + public SummaryFieldEntry(String key, Object value) { + this.key = key; + this.value = value; + } + + @Override + public String getKey() { return key; } + + @Override + public Object getValue() { return value; } + + @Override + public Object setValue(Object value) { throw new UnsupportedOperationException(); } + + } + + } + + /** Iterator over the field names in a SummaryData instance. Read only. */ + private static class SummaryDataFieldNameIterator extends SummaryDataIterator { + + SummaryDataFieldNameIterator(FastHit hit, + Iterator> fieldIterator, + int index) { + super(hit, fieldIterator, index); + advanceNext(); + } + + @Override + protected String toValue(Map.Entry field) { return field.getKey(); } + + } + } + + /** + * Abstract superclass of iterators over all the field content of a FastHit. + * This handles iterating over the iterators of Hit and the SummaryData instances of the FastHit, + * to provide a view of all the summary data of the FastHit. + * Iteration over fields of each piece of data (of Hit or a SummaryData instance) is delegated to the + * iterators of those types. + */ + private static abstract class SummaryIterator implements Iterator { + + private final FastHit hit; + + /** -1 means that the current iterator is the map iterator of the parent hit, not any summary data iterator */ + private int currentSummaryDataIndex = -1; + private Iterator currentIterator; + private VALUE previousReturned = null; + + public SummaryIterator(FastHit hit, Iterator mapFieldsIterator) { + this.hit = hit; + this.currentIterator = mapFieldsIterator; + } + + @Override + public boolean hasNext() { + if (currentIterator.hasNext()) return true; + return nextIterator(); + } + + @Override + public VALUE next() { + if (currentIterator.hasNext() || nextIterator()) return previousReturned = currentIterator.next(); + throw new NoSuchElementException(); + } + + @Override + public void remove() { + if (previousReturned == null) + throw new IllegalStateException(); + if ( ! ( currentIterator instanceof SummaryData.SummaryDataIterator)) + currentIterator.remove(); // remove from the map + if (hit.removedFields == null) + hit.removedFields = new HashSet<>(); + hit.removedFields.add(nameOf(previousReturned)); + previousReturned = null; + } + + protected abstract String nameOf(VALUE value); + protected abstract Iterator iteratorFor(SummaryData summary); + + /** Advanced to the next non-empty iterator, if any */ + private boolean nextIterator() { + while (++currentSummaryDataIndex < hit.summaries.size()) { + currentIterator = iteratorFor(hit.summaries.get(currentSummaryDataIndex)); + if (currentIterator.hasNext()) + return true; + } + return false; + } + + } + + /** Iterator over all the field content of a FastHit */ + private static class FieldIterator extends SummaryIterator> { + + public FieldIterator(FastHit hit, Iterator> mapFieldsIterator) { + super(hit, mapFieldsIterator); + } + + @Override + protected String nameOf(Map.Entry value) { + return value.getKey(); + } + + @Override + protected Iterator> iteratorFor(SummaryData summary) { + return summary.fieldIterator(); + } + + } + + /** Iterator over all the field names stored in a FastHit */ + private static class FieldNameIterator extends SummaryIterator { + + public FieldNameIterator(FastHit hit, Iterator mapFieldNamesIterator) { + super(hit, mapFieldNamesIterator); + } + + @Override + protected String nameOf(String value) { + return value; + } + + @Override + protected Iterator iteratorFor(SummaryData summary) { + return summary.fieldNameIterator(); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 0fed379d446..f13fb2e88f4 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -151,11 +151,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //-------------- Generic property containers -------------------------------- - /** - * The synchronous view of the JDisc request causing this query. - * - * @since 5.1 - */ + /** The synchronous view of the JDisc request causing this query */ private final HttpRequest httpRequest; /** The context, or null if there is no context */ @@ -195,7 +191,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private static QueryProfileType argumentType; static { - argumentType=new QueryProfileType("native"); + argumentType = new QueryProfileType("native"); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start")); @@ -622,7 +618,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * * @deprecated this is ignored */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 7 public void setCompress(boolean ignored) { } /** @@ -630,7 +626,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * * @deprecated this always returns false */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 7 public boolean getCompress() { return false; } /** Returns a string describing this query */ diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 0664720c7d7..15c148b7db7 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -401,14 +401,9 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable fields() { return getUnmodifiableFieldMap(); } + public Map fields() { return getUnmodifiableFieldMap(); } /** Allocate room for the given number of fields to avoid resizing. */ public void reserve(int minSize) { @@ -424,14 +419,10 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable> fieldIterator() { return getFieldMap().entrySet().iterator(); } + /** Returns a modifiable iterator over the fields of this */ + public Iterator> fieldIterator() { return getFieldMap().entrySet().iterator(); } - /** Returns a field value */ + /** Returns a field value or null if not present */ public Object getField(String value) { return fields != null ? fields.get(value) : null; } /** Removes all fields of this */ @@ -457,10 +448,22 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable fieldMap = getFieldMap(); Object value = fieldMap.remove(oldKey); @@ -468,7 +471,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable getFieldMap() { - return getFieldMap(16); + return getFieldMap(2); } private Map getFieldMap(int minSize) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index eb1b3a68219..a20949e3dfd 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -21,14 +21,14 @@ import com.yahoo.yolean.chain.Provides; /** * Minimal combinator for YQL+ syntax and heuristically parsed user queries. * - * @author Steinar Knutsen - * @since 5.1.28 + * @author Steinar Knutsen */ @Beta @Provides(MinimalQueryInserter.EXTERNAL_YQL) @Before(PhaseNames.TRANSFORMED_QUERY) @After("com.yahoo.prelude.statistics.StatisticsSearcher") public class MinimalQueryInserter extends Searcher { + public static final String EXTERNAL_YQL = "ExternalYql"; public static final CompoundName YQL = new CompoundName("yql"); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 9b2de75e272..83871b559b4 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -1,167 +1,323 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; - +import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.result.Hit; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; -import com.yahoo.slime.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.*; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; - +import static org.junit.Assert.assertTrue; public class SlimeSummaryTestCase { + private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg"; + private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg"; + @Test public void testDecodingEmpty() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul); - byte[] docsum = makeEmptyDocsum(); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("short_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("byte_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("float_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("double_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("int64_field"), equalTo(NanNumber.NaN)); - assertThat(hit.getField("string_field"), equalTo("")); - assertThat(hit.getField("data_field"), instanceOf(RawData.class)); - assertThat(hit.getField("data_field").toString(), equalTo("")); - assertThat(hit.getField("longstring_field"), equalTo((Object)"")); - assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); - assertThat(hit.getField("longdata_field").toString(), equalTo("")); - assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); - assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); - // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor + assertNull(docsum.lazyDecode("default", emptySummary(), hit)); + assertNull(hit.getField("integer_field")); + assertNull(hit.getField("short_field")); + assertNull(hit.getField("byte_field")); + assertNull(hit.getField("float_field")); + assertNull(hit.getField("double_field")); + assertNull(hit.getField("int64_field")); + assertNull(hit.getField("string_field")); + assertNull(hit.getField("data_field")); + assertNull(hit.getField("data_field")); + assertNull(hit.getField("longstring_field")); + assertNull(hit.getField("longdata_field")); + assertNull(hit.getField("longdata_field")); + assertNull(hit.getField("xmlstring_field")); + assertNull(hit.getField("xmlstring_field")); + assertNull(hit.getField("jsonstring_field")); assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); - } - @Test - public void testDecodingEmptyWithoutForcedFill() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false))); - byte[] docsum = makeEmptyDocsum(); + public void testDecodingEmptyWithLegacyEmulation() { + LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(null)); - assertThat(hit.getField("short_field"), equalTo(null)); - assertThat(hit.getField("byte_field"), equalTo(null)); - assertThat(hit.getField("float_field"), equalTo(null)); - assertThat(hit.getField("double_field"), equalTo(null)); - assertThat(hit.getField("int64_field"), equalTo(null)); - assertThat(hit.getField("string_field"), equalTo(null)); - assertThat(hit.getField("data_field"), equalTo(null)); - assertThat(hit.getField("data_field"), equalTo(null)); - assertThat(hit.getField("longstring_field"), equalTo(null)); - assertThat(hit.getField("longdata_field"), equalTo(null)); - assertThat(hit.getField("longdata_field"), equalTo(null)); - assertThat(hit.getField("xmlstring_field"), equalTo(null)); - assertThat(hit.getField("xmlstring_field"), equalTo(null)); - assertThat(hit.getField("jsonstring_field"), equalTo(null)); + assertNull(docsum.lazyDecode("default", emptySummary(), hit)); + assertEquals(NanNumber.NaN, hit.getField("integer_field")); + assertEquals(NanNumber.NaN, hit.getField("short_field")); + assertEquals(NanNumber.NaN, hit.getField("byte_field")); + assertEquals(NanNumber.NaN, hit.getField("float_field")); + assertEquals(NanNumber.NaN, hit.getField("double_field")); + assertEquals(NanNumber.NaN, hit.getField("int64_field")); + assertEquals("", hit.getField("string_field")); + assertEquals(RawData.class, hit.getField("data_field").getClass()); + assertEquals("", hit.getField("data_field").toString()); + assertEquals("", hit.getField("longstring_field")); + assertEquals(RawData.class, hit.getField("longdata_field").getClass()); + assertEquals("", hit.getField("longdata_field").toString()); + assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); + assertEquals("", hit.getField("xmlstring_field").toString()); + // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); + assertEquals("", hit.getField("jsonstring_field").toString()); + // Empty tensors are represented by null because we don't have type information here to create the right empty tensor assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } - private byte[] makeEmptyDocsum() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - return encode((slime)); - } - private byte [] encode(Slime slime) { - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - @Test public void testTimeout() { - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); - byte[] docsum = makeTimeout(); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", set.lazyDecode("default", docsum, hit)); + assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", + docsum.lazyDecode("default", timeoutSummary(), hit)); } @Test public void testDecoding() { Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); - - String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); - byte[] docsum = makeDocsum(tensor1, tensor2); + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); - assertNull(set.lazyDecode("default", docsum, hit)); - assertThat(hit.getField("integer_field"), equalTo(4)); - assertThat(hit.getField("short_field"), equalTo((short)2)); - assertThat(hit.getField("byte_field"), equalTo((byte)1)); - assertThat(hit.getField("float_field"), equalTo(4.5f)); - assertThat(hit.getField("double_field"), equalTo(8.75)); - assertThat(hit.getField("int64_field"), equalTo(8L)); - assertThat(hit.getField("string_field"), equalTo("string_value")); - assertThat(hit.getField("data_field"), instanceOf(RawData.class)); - assertThat(hit.getField("data_field").toString(), equalTo("data_value")); - assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); - assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); - assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value")); - assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); - assertThat(hit.getField("xmlstring_field").toString(), equalTo("xmlstring_value")); + assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit)); + assertEquals(4, hit.getField("integer_field")); + assertEquals((short)2, hit.getField("short_field")); + assertEquals((byte)1, hit.getField("byte_field")); + assertEquals(4.5F, hit.getField("float_field")); + assertEquals(8.75, hit.getField("double_field")); + assertEquals(8L, hit.getField("int64_field")); + assertEquals("string_value", hit.getField("string_field")); + assertEquals(RawData.class, hit.getField("data_field").getClass()); + assertEquals("data_value", hit.getField("data_field").toString()); + assertEquals("longstring_value", hit.getField("longstring_field")); + assertEquals(RawData.class, hit.getField("longdata_field").getClass()); + assertEquals("longdata_value", hit.getField("longdata_field").toString()); + assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); + assertEquals("xmlstring_value", hit.getField("xmlstring_field").toString()); if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); - assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}")); - assertThat(jstr.getParsedJSON(), notNullValue()); + assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); + assertNotNull(jstr.getParsedJSON()); - com.yahoo.data.access.Inspectable obj = jstr; - com.yahoo.data.access.Inspector value = obj.inspect(); - assertThat(value.field("foo").asLong(), equalTo(1L)); - assertThat(value.field("bar").asLong(), equalTo(2L)); + com.yahoo.data.access.Inspector value = jstr.inspect(); + assertEquals(1L, value.field("foo").asLong()); + assertEquals(2L, value.field("bar").asLong()); } else { StructuredData sdata = (StructuredData) hit.getField("jsonstring_field"); - assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}")); + assertEquals("{\"foo\":1,\"bar\":2}", sdata.toJson()); - com.yahoo.data.access.Inspectable obj = sdata; - com.yahoo.data.access.Inspector value = obj.inspect(); - assertThat(value.field("foo").asLong(), equalTo(1L)); - assertThat(value.field("bar").asLong(), equalTo(2L)); + com.yahoo.data.access.Inspector value = sdata.inspect(); + assertEquals(1L, value.field("foo").asLong()); + assertEquals(2L, value.field("bar").asLong()); } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0)); + @Test + public void testFieldAccessAPI() { + DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); + DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); + FastHit hit = new FastHit(); + Map expected = new HashMap<>(); + + assertFields(expected, hit); + + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + assertFields(expected, hit); + + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + assertFields(expected, hit); + + hit.removeField("short_field"); + expected.remove("short_field"); + assertFields(expected, hit); + + hit.setField("string", "hello"); + expected.put("string", "hello"); + assertFields(expected, hit); + + hit.setField("short_field", 3.8F); + expected.put("short_field", 3.8F); + assertFields(expected, hit); + + hit.removeField("string"); + expected.remove("string"); + assertFields(expected, hit); + + hit.removeField("integer_field"); + hit.removeField("double_field"); + expected.remove("integer_field"); + expected.remove("double_field"); + assertFields(expected, hit); + + hit.clearFields(); + expected.clear(); + assertFields(expected, hit); + + // --- Re-populate + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + hit.setField("string1", "hello"); + hit.setField("string2", "hello"); + expected.put("string1", "hello"); + expected.put("string2", "hello"); + assertFields(expected, hit); + + Set keys = hit.fieldKeys(); + assertTrue(keys.remove("integer_field")); + expected.remove("integer_field"); + assertTrue(keys.remove("string2")); + expected.remove("string2"); + assertFields(expected, hit); + assertFalse(keys.remove("notpresent")); + + assertTrue(keys.retainAll(ImmutableSet.of("string1", "notpresent", "double_field"))); + expected.remove("short_field"); + expected.remove("float_field"); + assertFields(expected, hit); + + Iterator keyIterator = keys.iterator(); + assertEquals("string1", keyIterator.next()); + keyIterator.remove(); + expected.remove("string1"); + assertFields(expected, hit); + + assertEquals("double_field", keyIterator.next()); + keyIterator.remove(); + expected.remove("double_field"); + assertFields(expected, hit); + + // --- Re-populate + partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); + expected.put("integer_field", 4); + expected.put("short_field", (short) 2); + partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); + expected.put("float_field", 4.5F); + expected.put("double_field", 8.75D); + hit.setField("string", "hello"); + expected.put("string", "hello"); + assertFields(expected, hit); + + Iterator> fieldIterator = hit.fieldIterator(); + assertEquals("string", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("string"); + assertFields(expected, hit); + + fieldIterator.next(); + assertEquals("short_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("short_field"); + assertFields(expected, hit); + + fieldIterator.next(); + assertEquals("double_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("double_field"); + assertFields(expected, hit); + + fieldIterator = hit.fieldIterator(); + assertEquals("float_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("float_field"); + assertFields(expected, hit); + + assertEquals("integer_field", fieldIterator.next().getKey()); + fieldIterator.remove(); + expected.remove("integer_field"); + assertFields(expected, hit); } - private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) { + + /** Asserts that the expected fields are what is returned from every access method of Hit */ + private void assertFields(Map expected, Hit hit) { + // fieldKeys + int fieldNameIteratorFieldCount = 0; + for (Iterator i = hit.fieldKeys().iterator(); i.hasNext(); ) { + fieldNameIteratorFieldCount++; + assertTrue(expected.containsKey(i.next())); + } + assertEquals(expected.size(), fieldNameIteratorFieldCount); + assertEquals(expected.keySet(), hit.fieldKeys()); + // getField + for (Map.Entry field : expected.entrySet()) + assertEquals(field.getValue(), hit.getField(field.getKey())); + // fields + assertEquals(expected, hit.fields()); + // fieldIterator + int fieldIteratorFieldCount = 0; + for (Iterator> i = hit.fieldIterator(); i.hasNext(); ) { + fieldIteratorFieldCount++; + Map.Entry field = i.next(); + assertEquals(field.getValue(), expected.get(field.getKey())); + } + assertEquals(expected.size(), fieldIteratorFieldCount); + } + + private byte[] emptySummary() { + Slime slime = new Slime(); + slime.setObject(); + return encode((slime)); + } + + private byte [] timeoutSummary() { + Slime slime = new Slime(); + slime.setString("Timed out...."); + return encode((slime)); + } + + private byte[] partialSummary1() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + return encode((slime)); + } + + private byte[] partialSummary2() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + return encode((slime)); + } + + private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); docsum.setLong("integer_field", 4); @@ -185,10 +341,24 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte [] makeTimeout() { - Slime slime = new Slime(); - slime.setString("Timed out...."); - return encode((slime)); + private byte[] encode(Slime slime) { + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); + } + + private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg new file mode 100644 index 00000000000..5aa5c84d936 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg @@ -0,0 +1,10 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 1 +documentdb[0].summaryclass[0].fields[2] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name short_field +documentdb[0].summaryclass[0].fields[1].type short diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg new file mode 100644 index 00000000000..bc870a63d66 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg @@ -0,0 +1,12 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 2 +documentdb[0].summaryclass[0].fields[3] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name float_field +documentdb[0].summaryclass[0].fields[1].type float +documentdb[0].summaryclass[0].fields[2].name double_field +documentdb[0].summaryclass[0].fields[2].type double -- cgit v1.2.3