diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-05-07 15:15:55 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-07 15:15:55 +0200 |
commit | 321ee8d0bffef15faf00474e3518009bab58f4cf (patch) | |
tree | f80025c882ed39c44d0a52144486e5fb9d326c1b | |
parent | 4da4fd9a431601a265c4ea968f593baad94a9149 (diff) |
Revert "Bratseth/lazy summary decoding"
8 files changed, 149 insertions, 907 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index e20340a03c0..d50006fb82c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -11,7 +11,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; /** * A docsum definition which knows how to decode a certain class of document @@ -67,15 +66,9 @@ public class DocsumDefinition { return fields.get(fieldIndex); } - /** Returns the field with this name, or null if none */ - public DocsumField getField(String fieldName) { - Integer index = fieldNameToIndex.get(fieldName); - if (index == null) return null; - return getField(index); - } - - public Set<String> fieldNames() { - return fieldNameToIndex.keySet(); + /** Returns the index of a field name */ + public Integer getFieldIndex(String fieldName) { + return fieldNameToIndex.get(fieldName); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 298bd839393..f1f5e2b7403 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.data.access.ObjectTraverser; import com.yahoo.document.GlobalId; import com.yahoo.fs4.QueryPacketData; import com.yahoo.net.URI; @@ -9,17 +8,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.Relevance; import com.yahoo.data.access.Inspector; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; - /** * A regular hit from a Vespa backend * @@ -46,24 +34,6 @@ public class FastHit extends Hit { private transient CacheKey cacheKey = null; /** - * Summaries added to this hit which are not yet decoded into fields. - * Fields are resolved by returning the first non-null value found by - * 1) the field value from the Map of fields in the Hit supertype, and - * 2) each of the summaries, in the order of the list (which is the add order). - * This ensures that values set from code overwrites any value received as - * summary data. - * - * The reason we keep this rather than eagerly decoding into a the field map - * is to reduce garbage collection and decoding cost, with the assumption - * that most fields passes through the container with no processing most - * of the time. - */ - private List<SummaryData> summaries = new ArrayList<>(1); - - /** Removed field values, which should therefore not be returned if present in summary data */ - private Set<String> removedFields = null; - - /** * Creates an empty and temporarily invalid summary hit */ public FastHit() { } @@ -158,9 +128,16 @@ public class FastHit extends Hit { /** For internal use */ public void addSummary(DocsumDefinition docsumDef, Inspector value) { - if (removedFields != null) - removedFields.removeAll(docsumDef.fieldNames()); - summaries.add(new SummaryData(this, docsumDef, value, summaries.size())); + reserve(docsumDef.getFieldCount()); + for (DocsumField field : docsumDef.getFields()) { + String fieldName = field.getName(); + Inspector f = value.field(fieldName); + if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) { + if (super.getField(fieldName) == null) { + setField(fieldName, field.convert(f)); + } + } + } } /** @@ -195,132 +172,8 @@ public class FastHit extends Hit { * </ul> */ @Override - public Object getField(String name) { - Object value = super.getField(name); - if (value != null) return value; - value = getSummaryValue(name); - if (value != null) - super.setField(name, value); - return value; - } - - @Override - public Object setField(String name, Object value) { - if (removedFields != null) { - if (removedFields.remove(name)) { - if (removedFields.isEmpty()) - removedFields = null; - } - } - Object oldValue = super.setField(name, value); - if (oldValue != null) return oldValue; - return getSummaryValue(name); - } - - /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ - @Override - public Map<String, Object> fields() { - Map<String, Object> fields = new HashMap<>(); - for (Iterator<Map.Entry<String, Object>> i = fieldIterator(); i.hasNext(); ) { - Map.Entry<String, Object> field = i.next(); - fields.put(field.getKey(), field.getValue()); - } - return fields; - } - - /** Returns a modifiable iterator over the fields of this */ - @Override - public Iterator<Map.Entry<String, Object>> fieldIterator() { - return new FieldIterator(this, super.fieldIterator()); - } - - /** Returns a modifiable iterator over the field names of this */ - Iterator<String> fieldNameIterator() { - return new FieldNameIterator(this, super.fieldKeys().iterator()); - } - - /** Removes all fields of this */ - @Override - public void clearFields() { - summaries.clear(); - if (removedFields != null) - removedFields = null; - super.clearFields(); - } - - /** - * Removes a field from this - * - * @return the removed value of the field, or null if none - */ - @Override - public Object removeField(String name) { - Object removedValue = super.removeField(name); - if (removedValue == null) - removedValue = getSummaryValue(name); - - if (removedValue != null) { - if (removedFields == null) - removedFields = new HashSet<>(2); - removedFields.add(name); - } - - return removedValue; - } - - /** - * Returns the keys of the fields of this hit as a modifiable view. - * This follows the rules of key sets returned from maps: Key removals are reflected - * in the map, add and addAll is not supported. - */ - @Override - public Set<String> fieldKeys() { - return new FieldSet(this); - } - - private Set<String> mapFieldKeys() { - return super.fieldKeys(); - } - - /** Returns whether this field is present <b>in the field map in the parent hit</b> */ - // Note: If this is made public it must be changed to also check the summary data - // (and internal usage must change to another method). - @Override - protected boolean hasField(String name) { - return super.hasField(name); - } - - /** Returns whether any fields are present <b>in the field map in the parent hit</b> */ - // Note: If this is made public it must be changed to also check the summary data - // (and internal usage must change to another method). - @Override - protected boolean hasFields() { - return super.hasFields(); - } - - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - @Deprecated - @Override - @SuppressWarnings("deprecation") - public void changeFieldKey(String oldKey, String newKey) { - Object value = removeField(oldKey); - if (value != null) - setField(newKey, value); - } - - private Object getSummaryValue(String name) { - if (removedFields != null && removedFields.contains(name)) - return null; - for (SummaryData summaryData : summaries) { - Object value = summaryData.getField(name); - if (value != null) return value; - } - return null; + public Object getField(String key) { + return super.getField(key); } @Override @@ -339,8 +192,6 @@ public class FastHit extends Hit { } } - /** @deprecated do not use */ - @Deprecated // TODO: Make private on Vespa 7 public static String asHexString(GlobalId gid) { StringBuilder sb = new StringBuilder(); byte[] rawGid = gid.getRawId(); @@ -354,415 +205,4 @@ public class FastHit extends Hit { return sb.toString(); } - /** A set view of all the field names in this hit. Add/addAll is not supported but remove is. */ - private static class FieldSet implements Set<String> { - - // A set implementation which tries to avoid creating an actual set when possible. - // With more work this could be optimized to avoid creating the set in additional cases. - - private final FastHit hit; - - /** The computed set of fields. Lazily created as it is not always needed. */ - private Set<String> fieldSet = null; - - public FieldSet(FastHit hit) { - this.hit = hit; - } - - @Override - public int size() { - return createSet().size(); - } - - @Override - public boolean isEmpty() { - if ( ! hit.hasFields() && hit.summaries.isEmpty()) return true; - return createSet().isEmpty(); - } - - @Override - public boolean contains(Object o) { - String field = (String)o; - if (hit.hasField(field)) return true; - return createSet().contains(field); - } - - @Override - public Object[] toArray() { - return createSet().toArray(); - } - - @Override - public <T> T[] toArray(T[] a) { - return createSet().toArray(a); - } - - @Override - public boolean add(String s) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean remove(Object o) { - String field = (String)o; - boolean removed = hit.removeField(field) != null; - if (fieldSet != null) - fieldSet.remove(field); - return removed; - } - - @Override - public boolean containsAll(Collection<?> c) { - for (Object field : c) - if ( ! contains(field)) - return false; - return true; - } - - @Override - public boolean addAll(Collection<? extends String> c) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean retainAll(Collection<?> c) { - Set<?> toRetain = c instanceof Set<?> ? (Set<?>)c : new HashSet<Object>(c); - boolean anyRemoved = false; - for (Iterator<String> i = iterator(); i.hasNext(); ) { - String field = i.next(); - if (toRetain.contains(field)) continue; - - i.remove(); - anyRemoved = true; - } - return anyRemoved; - } - - @Override - public boolean removeAll(Collection<?> c) { - boolean anyRemoved = false; - for (Object field : c) - if (remove(field)) - anyRemoved = true; - return anyRemoved; - } - - @Override - public void clear() { - hit.clearFields(); - fieldSet = null; - } - - @Override - public Iterator<String> iterator() { - return hit.fieldNameIterator(); - } - - @Override - public int hashCode() { - return createSet().hashCode(); - } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof Set)) return false; - return createSet().equals(o); - } - - @Override - public String toString() { - return createSet().toString(); - } - - private Set<String> createSet() { - if (this.fieldSet != null) return this.fieldSet; - if ( ! hit.hasFields() && hit.summaries.isEmpty()) return Collections.emptySet(); // shortcut - - Set<String> fields = new HashSet<>(); - if (hit.hasFields()) - fields.addAll(hit.mapFieldKeys()); - - for (SummaryData summaryData : hit.summaries) - summaryData.data.traverse((ObjectTraverser)(name, __) -> fields.add(name)); - if (hit.removedFields != null) - fields.removeAll(hit.removedFields); - - this.fieldSet = fields; - - return fields; - } - - } - - /** Summary data (values of a number of fields) received for this hit */ - private static class SummaryData { - - private final FastHit hit; - private final DocsumDefinition type; - private final Inspector data; - - /** The index of this summary in the list of summaries added to this */ - private final int index; - - SummaryData(FastHit hit, DocsumDefinition type, Inspector data, int index) { - this.hit = hit; - this.type = type; - this.data = data; - this.index = index; - } - - Object getField(String name) { - DocsumField fieldType = type.getField(name); - if (fieldType == null) return null; - Inspector fieldValue = data.field(name); - if ( ! fieldValue.valid() && ! fieldType.getEmulConfig().forceFillEmptyFields()) return null; - return fieldType.convert(fieldValue); - } - - /** Decodes the given summary into the field map in the parent */ - private void decodeInto(FastHit hit) { - hit.reserve(type.getFieldCount()); - for (DocsumField field : type.getFields()) { - String fieldName = field.getName(); - Inspector f = data.field(fieldName); - if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) { - if (hit.getField(fieldName) == null) - hit.setField(fieldName, field.convert(f)); - } - } - } - - private Iterator<Map.Entry<String, Object>> fieldIterator() { - return new SummaryDataFieldIterator(hit, type, data.fields().iterator(), index); - } - - private Iterator<String> fieldNameIterator() { - return new SummaryDataFieldNameIterator(hit, data.fields().iterator(), index); - } - - /** - * Abstract superclass of iterators over SummaryData content which takes care of skipping unknown, - * removed and already returned fields. Read only. - */ - private static abstract class SummaryDataIterator<VALUE> implements Iterator<VALUE> { - - private final FastHit hit; - private final Iterator<Map.Entry<String, Inspector>> fieldIterator; - private final int index; - - /** The next value or null if none, eagerly read because we need to skip removed and overwritten values */ - private VALUE next; - - SummaryDataIterator(FastHit hit, - Iterator<Map.Entry<String, Inspector>> fieldIterator, - int index) { - this.hit = hit; - this.fieldIterator = fieldIterator; - this.index = index; - } - - @Override - public boolean hasNext() { - return next != null; - } - - @Override - public VALUE next() { - if (next == null) throw new NoSuchElementException(); - - VALUE returnValue = next; - advanceNext(); - return returnValue; - } - - protected abstract VALUE toValue(Map.Entry<String, Inspector> field); - - protected void advanceNext() { - while (fieldIterator.hasNext()) { - Map.Entry<String, Inspector> nextEntry = fieldIterator.next(); - String fieldName = nextEntry.getKey(); - next = toValue(nextEntry); - if ( next != null && - ! hit.hasField(fieldName) && - ! isRemoved(fieldName) && - ! isAlreadyReturned(fieldName)) - return; - } - next = null; - } - - private boolean isRemoved(String fieldName) { - return hit.removedFields != null && hit.removedFields.contains(fieldName); - } - - private boolean isAlreadyReturned(String fieldName) { - for (int i = 0; i < index; i++) { - if (hit.summaries.get(i).type.fieldNames().contains(fieldName)) - return true; - } - return false; - } - - } - - - /** Iterator over the fields in a SummaryData instance. Read only. */ - private static class SummaryDataFieldIterator extends SummaryDataIterator<Map.Entry<String, Object>> { - - private final DocsumDefinition type; - - SummaryDataFieldIterator(FastHit hit, - DocsumDefinition type, - Iterator<Map.Entry<String, Inspector>> fieldIterator, - int index) { - super(hit, fieldIterator, index); - this.type = type; - advanceNext(); - } - - @Override - protected Map.Entry<String, Object> toValue(Map.Entry<String, Inspector> field) { - DocsumField fieldType = type.getField(field.getKey()); - if (fieldType == null) return null; // type and content mismatch: May happen during reconfig - return new SummaryFieldEntry(field.getKey(), fieldType.convert(field.getValue())); - } - - private static final class SummaryFieldEntry implements Map.Entry<String, Object> { - - private final String key; - private final Object value; - - public SummaryFieldEntry(String key, Object value) { - this.key = key; - this.value = value; - } - - @Override - public String getKey() { return key; } - - @Override - public Object getValue() { return value; } - - @Override - public Object setValue(Object value) { throw new UnsupportedOperationException(); } - - } - - } - - /** Iterator over the field names in a SummaryData instance. Read only. */ - private static class SummaryDataFieldNameIterator extends SummaryDataIterator<String> { - - SummaryDataFieldNameIterator(FastHit hit, - Iterator<Map.Entry<String, Inspector>> fieldIterator, - int index) { - super(hit, fieldIterator, index); - advanceNext(); - } - - @Override - protected String toValue(Map.Entry<String, Inspector> field) { return field.getKey(); } - - } - } - - /** - * Abstract superclass of iterators over all the field content of a FastHit. - * This handles iterating over the iterators of Hit and the SummaryData instances of the FastHit, - * to provide a view of all the summary data of the FastHit. - * Iteration over fields of each piece of data (of Hit or a SummaryData instance) is delegated to the - * iterators of those types. - */ - private static abstract class SummaryIterator<VALUE> implements Iterator<VALUE> { - - private final FastHit hit; - - /** -1 means that the current iterator is the map iterator of the parent hit, not any summary data iterator */ - private int currentSummaryDataIndex = -1; - private Iterator<VALUE> currentIterator; - private VALUE previousReturned = null; - - public SummaryIterator(FastHit hit, Iterator<VALUE> mapFieldsIterator) { - this.hit = hit; - this.currentIterator = mapFieldsIterator; - } - - @Override - public boolean hasNext() { - if (currentIterator.hasNext()) return true; - return nextIterator(); - } - - @Override - public VALUE next() { - if (currentIterator.hasNext() || nextIterator()) return previousReturned = currentIterator.next(); - throw new NoSuchElementException(); - } - - @Override - public void remove() { - if (previousReturned == null) - throw new IllegalStateException(); - if ( ! ( currentIterator instanceof SummaryData.SummaryDataIterator)) - currentIterator.remove(); // remove from the map - if (hit.removedFields == null) - hit.removedFields = new HashSet<>(); - hit.removedFields.add(nameOf(previousReturned)); - previousReturned = null; - } - - protected abstract String nameOf(VALUE value); - protected abstract Iterator<VALUE> iteratorFor(SummaryData summary); - - /** Advanced to the next non-empty iterator, if any */ - private boolean nextIterator() { - while (++currentSummaryDataIndex < hit.summaries.size()) { - currentIterator = iteratorFor(hit.summaries.get(currentSummaryDataIndex)); - if (currentIterator.hasNext()) - return true; - } - return false; - } - - } - - /** Iterator over all the field content of a FastHit */ - private static class FieldIterator extends SummaryIterator<Map.Entry<String, Object>> { - - public FieldIterator(FastHit hit, Iterator<Map.Entry<String, Object>> mapFieldsIterator) { - super(hit, mapFieldsIterator); - } - - @Override - protected String nameOf(Map.Entry<String, Object> value) { - return value.getKey(); - } - - @Override - protected Iterator<Map.Entry<String, Object>> iteratorFor(SummaryData summary) { - return summary.fieldIterator(); - } - - } - - /** Iterator over all the field names stored in a FastHit */ - private static class FieldNameIterator extends SummaryIterator<String> { - - public FieldNameIterator(FastHit hit, Iterator<String> mapFieldNamesIterator) { - super(hit, mapFieldNamesIterator); - } - - @Override - protected String nameOf(String value) { - return value; - } - - @Override - protected Iterator<String> iteratorFor(SummaryData summary) { - return summary.fieldNameIterator(); - } - - } - } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index f13fb2e88f4..0fed379d446 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -151,7 +151,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //-------------- Generic property containers -------------------------------- - /** The synchronous view of the JDisc request causing this query */ + /** + * The synchronous view of the JDisc request causing this query. + * + * @since 5.1 + */ private final HttpRequest httpRequest; /** The context, or null if there is no context */ @@ -191,7 +195,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private static QueryProfileType argumentType; static { - argumentType = new QueryProfileType("native"); + argumentType=new QueryProfileType("native"); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start")); @@ -618,7 +622,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * * @deprecated this is ignored */ - @Deprecated // TODO: Remove on Vespa 7 + @Deprecated public void setCompress(boolean ignored) { } /** @@ -626,7 +630,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * * @deprecated this always returns false */ - @Deprecated // TODO: Remove on Vespa 7 + @Deprecated public boolean getCompress() { return false; } /** Returns a string describing this query */ diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 15c148b7db7..0664720c7d7 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -401,9 +401,14 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Returns the name of the source creating this hit */ public String getSource() { return source; } - /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ + /** + * Returns the fields of this as a read-only map. This is more costly than the preferred iterator(), as + * it uses Collections.unmodifiableMap() + * + * @return An readonly map of the fields + */ // TODO Should it be deprecated ? - public Map<String, Object> fields() { return getUnmodifiableFieldMap(); } + public final Map<String, Object> fields() { return getUnmodifiableFieldMap(); } /** Allocate room for the given number of fields to avoid resizing. */ public void reserve(int minSize) { @@ -419,10 +424,14 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return getFieldMap().put(key, value); } - /** Returns a modifiable iterator over the fields of this */ - public Iterator<Map.Entry<String, Object>> fieldIterator() { return getFieldMap().entrySet().iterator(); } + /** + * Returns an iterator over the fields of this + * + * @return an iterator for traversing the fields of this hit + */ + public final Iterator<Map.Entry<String,Object>> fieldIterator() { return getFieldMap().entrySet().iterator(); } - /** Returns a field value or null if not present */ + /** Returns a field value */ public Object getField(String value) { return fields != null ? fields.get(value) : null; } /** Removes all fields of this */ @@ -448,22 +457,10 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return getFieldMap().keySet(); } - protected boolean hasField(String name) { - return fields != null && fields.containsKey(name); - } - - /** Returns whether any fields are set in this */ - protected boolean hasFields() { - return fields != null && ! fields.isEmpty(); - } - /** * Changes the key under which a value is found. This is useful because it allows keys to be changed * without accessing the value (which may be lazily created). - * - * @deprecated do not use */ - @Deprecated // TODO: Remove on Vespa 7 public void changeFieldKey(String oldKey, String newKey) { Map<String,Object> fieldMap = getFieldMap(); Object value = fieldMap.remove(oldKey); @@ -471,7 +468,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } private Map<String, Object> getFieldMap() { - return getFieldMap(2); + return getFieldMap(16); } private Map<String, Object> getFieldMap(int minSize) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index a20949e3dfd..eb1b3a68219 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -21,14 +21,14 @@ import com.yahoo.yolean.chain.Provides; /** * Minimal combinator for YQL+ syntax and heuristically parsed user queries. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.28 */ @Beta @Provides(MinimalQueryInserter.EXTERNAL_YQL) @Before(PhaseNames.TRANSFORMED_QUERY) @After("com.yahoo.prelude.statistics.StatisticsSearcher") public class MinimalQueryInserter extends Searcher { - public static final String EXTERNAL_YQL = "ExternalYql"; public static final CompoundName YQL = new CompoundName("yql"); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 83871b559b4..9b2de75e272 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -1,323 +1,167 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.google.common.collect.ImmutableSet; + import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.search.result.Hit; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; +import com.yahoo.slime.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import com.yahoo.slime.BinaryFormat; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import org.junit.Test; - +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -public class SlimeSummaryTestCase { - private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; - private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg"; - private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg"; +public class SlimeSummaryTestCase { @Test public void testDecodingEmpty() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul); + byte[] docsum = makeEmptyDocsum(); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertNull(hit.getField("integer_field")); - assertNull(hit.getField("short_field")); - assertNull(hit.getField("byte_field")); - assertNull(hit.getField("float_field")); - assertNull(hit.getField("double_field")); - assertNull(hit.getField("int64_field")); - assertNull(hit.getField("string_field")); - assertNull(hit.getField("data_field")); - assertNull(hit.getField("data_field")); - assertNull(hit.getField("longstring_field")); - assertNull(hit.getField("longdata_field")); - assertNull(hit.getField("longdata_field")); - assertNull(hit.getField("xmlstring_field")); - assertNull(hit.getField("xmlstring_field")); - assertNull(hit.getField("jsonstring_field")); + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("short_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("byte_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("float_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("double_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("int64_field"), equalTo(NanNumber.NaN)); + assertThat(hit.getField("string_field"), equalTo("")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); + // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); + assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); + // Empty tensors are represented by null because we don't have type information here to create the right empty tensor assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } + private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); + } + @Test - public void testDecodingEmptyWithLegacyEmulation() { - LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); + public void testDecodingEmptyWithoutForcedFill() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false))); + byte[] docsum = makeEmptyDocsum(); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertEquals(NanNumber.NaN, hit.getField("integer_field")); - assertEquals(NanNumber.NaN, hit.getField("short_field")); - assertEquals(NanNumber.NaN, hit.getField("byte_field")); - assertEquals(NanNumber.NaN, hit.getField("float_field")); - assertEquals(NanNumber.NaN, hit.getField("double_field")); - assertEquals(NanNumber.NaN, hit.getField("int64_field")); - assertEquals("", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("", hit.getField("data_field").toString()); - assertEquals("", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("", hit.getField("xmlstring_field").toString()); - // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertEquals("", hit.getField("jsonstring_field").toString()); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(null)); + assertThat(hit.getField("short_field"), equalTo(null)); + assertThat(hit.getField("byte_field"), equalTo(null)); + assertThat(hit.getField("float_field"), equalTo(null)); + assertThat(hit.getField("double_field"), equalTo(null)); + assertThat(hit.getField("int64_field"), equalTo(null)); + assertThat(hit.getField("string_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("longstring_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("jsonstring_field"), equalTo(null)); assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); } + private byte[] makeEmptyDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + return encode((slime)); + } + private byte [] encode(Slime slime) { + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + @Test public void testTimeout() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + byte[] docsum = makeTimeout(); FastHit hit = new FastHit(); - assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", - docsum.lazyDecode("default", timeoutSummary(), hit)); + assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", set.lazyDecode("default", docsum, hit)); } @Test public void testDecoding() { Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); + + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + byte[] docsum = makeDocsum(tensor1, tensor2); FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit)); - assertEquals(4, hit.getField("integer_field")); - assertEquals((short)2, hit.getField("short_field")); - assertEquals((byte)1, hit.getField("byte_field")); - assertEquals(4.5F, hit.getField("float_field")); - assertEquals(8.75, hit.getField("double_field")); - assertEquals(8L, hit.getField("int64_field")); - assertEquals("string_value", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("data_value", hit.getField("data_field").toString()); - assertEquals("longstring_value", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("longdata_value", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("<tag>xmlstring_value</tag>", hit.getField("xmlstring_field").toString()); + assertNull(set.lazyDecode("default", docsum, hit)); + assertThat(hit.getField("integer_field"), equalTo(4)); + assertThat(hit.getField("short_field"), equalTo((short)2)); + assertThat(hit.getField("byte_field"), equalTo((byte)1)); + assertThat(hit.getField("float_field"), equalTo(4.5f)); + assertThat(hit.getField("double_field"), equalTo(8.75)); + assertThat(hit.getField("int64_field"), equalTo(8L)); + assertThat(hit.getField("string_field"), equalTo("string_value")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("data_value")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("<tag>xmlstring_value</tag>")); if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); - assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); - assertNotNull(jstr.getParsedJSON()); + assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}")); + assertThat(jstr.getParsedJSON(), notNullValue()); - com.yahoo.data.access.Inspector value = jstr.inspect(); - assertEquals(1L, value.field("foo").asLong()); - assertEquals(2L, value.field("bar").asLong()); + com.yahoo.data.access.Inspectable obj = jstr; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); } else { StructuredData sdata = (StructuredData) hit.getField("jsonstring_field"); - assertEquals("{\"foo\":1,\"bar\":2}", sdata.toJson()); + assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}")); - com.yahoo.data.access.Inspector value = sdata.inspect(); - assertEquals(1L, value.field("foo").asLong()); - assertEquals(2L, value.field("bar").asLong()); + com.yahoo.data.access.Inspectable obj = sdata; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); } - @Test - public void testFieldAccessAPI() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); - DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); - DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); - FastHit hit = new FastHit(); - Map<String, Object> expected = new HashMap<>(); - - assertFields(expected, hit); - - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - assertFields(expected, hit); - - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - assertFields(expected, hit); - - hit.removeField("short_field"); - expected.remove("short_field"); - assertFields(expected, hit); - - hit.setField("string", "hello"); - expected.put("string", "hello"); - assertFields(expected, hit); - - hit.setField("short_field", 3.8F); - expected.put("short_field", 3.8F); - assertFields(expected, hit); - - hit.removeField("string"); - expected.remove("string"); - assertFields(expected, hit); - - hit.removeField("integer_field"); - hit.removeField("double_field"); - expected.remove("integer_field"); - expected.remove("double_field"); - assertFields(expected, hit); - - hit.clearFields(); - expected.clear(); - assertFields(expected, hit); - - // --- Re-populate - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - hit.setField("string1", "hello"); - hit.setField("string2", "hello"); - expected.put("string1", "hello"); - expected.put("string2", "hello"); - assertFields(expected, hit); - - Set<String> keys = hit.fieldKeys(); - assertTrue(keys.remove("integer_field")); - expected.remove("integer_field"); - assertTrue(keys.remove("string2")); - expected.remove("string2"); - assertFields(expected, hit); - assertFalse(keys.remove("notpresent")); - - assertTrue(keys.retainAll(ImmutableSet.of("string1", "notpresent", "double_field"))); - expected.remove("short_field"); - expected.remove("float_field"); - assertFields(expected, hit); - - Iterator<String> keyIterator = keys.iterator(); - assertEquals("string1", keyIterator.next()); - keyIterator.remove(); - expected.remove("string1"); - assertFields(expected, hit); - - assertEquals("double_field", keyIterator.next()); - keyIterator.remove(); - expected.remove("double_field"); - assertFields(expected, hit); - - // --- Re-populate - partialDocsum1.lazyDecode("partial1", partialSummary1(), hit); - expected.put("integer_field", 4); - expected.put("short_field", (short) 2); - partialDocsum2.lazyDecode("partial2", partialSummary2(), hit); - expected.put("float_field", 4.5F); - expected.put("double_field", 8.75D); - hit.setField("string", "hello"); - expected.put("string", "hello"); - assertFields(expected, hit); - - Iterator<Map.Entry<String, Object>> fieldIterator = hit.fieldIterator(); - assertEquals("string", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("string"); - assertFields(expected, hit); - - fieldIterator.next(); - assertEquals("short_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("short_field"); - assertFields(expected, hit); - - fieldIterator.next(); - assertEquals("double_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("double_field"); - assertFields(expected, hit); - - fieldIterator = hit.fieldIterator(); - assertEquals("float_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("float_field"); - assertFields(expected, hit); - - assertEquals("integer_field", fieldIterator.next().getKey()); - fieldIterator.remove(); - expected.remove("integer_field"); - assertFields(expected, hit); - } - - - /** Asserts that the expected fields are what is returned from every access method of Hit */ - private void assertFields(Map<String, Object> expected, Hit hit) { - // fieldKeys - int fieldNameIteratorFieldCount = 0; - for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) { - fieldNameIteratorFieldCount++; - assertTrue(expected.containsKey(i.next())); - } - assertEquals(expected.size(), fieldNameIteratorFieldCount); - assertEquals(expected.keySet(), hit.fieldKeys()); - // getField - for (Map.Entry<String, Object> field : expected.entrySet()) - assertEquals(field.getValue(), hit.getField(field.getKey())); - // fields - assertEquals(expected, hit.fields()); - // fieldIterator - int fieldIteratorFieldCount = 0; - for (Iterator<Map.Entry<String, Object>> i = hit.fieldIterator(); i.hasNext(); ) { - fieldIteratorFieldCount++; - Map.Entry<String, Object> field = i.next(); - assertEquals(field.getValue(), expected.get(field.getKey())); - } - assertEquals(expected.size(), fieldIteratorFieldCount); - } - - private byte[] emptySummary() { - Slime slime = new Slime(); - slime.setObject(); - return encode((slime)); - } - - private byte [] timeoutSummary() { - Slime slime = new Slime(); - slime.setString("Timed out...."); - return encode((slime)); - } - - private byte[] partialSummary1() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setLong("integer_field", 4); - docsum.setLong("short_field", 2); - return encode((slime)); - } - - private byte[] partialSummary2() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setLong("integer_field", 4); - docsum.setDouble("float_field", 4.5); - docsum.setDouble("double_field", 8.75); - return encode((slime)); + private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); } - private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { + private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); docsum.setLong("integer_field", 4); @@ -341,24 +185,10 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte[] encode(Slime slime) { - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - - private DocsumDefinitionSet createDocsumDefinitionSet(String configID) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0)); - } - - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); + private byte [] makeTimeout() { + Slime slime = new Slime(); + slime.setString("Timed out...."); + return encode((slime)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg deleted file mode 100644 index 5aa5c84d936..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg +++ /dev/null @@ -1,10 +0,0 @@ -documentdb[1] -documentdb[0].name test -documentdb[0].summaryclass[1] -documentdb[0].summaryclass[0].name default -documentdb[0].summaryclass[0].id 1 -documentdb[0].summaryclass[0].fields[2] -documentdb[0].summaryclass[0].fields[0].name integer_field -documentdb[0].summaryclass[0].fields[0].type integer -documentdb[0].summaryclass[0].fields[1].name short_field -documentdb[0].summaryclass[0].fields[1].type short diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg deleted file mode 100644 index bc870a63d66..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg +++ /dev/null @@ -1,12 +0,0 @@ -documentdb[1] -documentdb[0].name test -documentdb[0].summaryclass[1] -documentdb[0].summaryclass[0].name default -documentdb[0].summaryclass[0].id 2 -documentdb[0].summaryclass[0].fields[3] -documentdb[0].summaryclass[0].fields[0].name integer_field -documentdb[0].summaryclass[0].fields[0].type integer -documentdb[0].summaryclass[0].fields[1].name float_field -documentdb[0].summaryclass[0].fields[1].type float -documentdb[0].summaryclass[0].fields[2].name double_field -documentdb[0].summaryclass[0].fields[2].type double |