Merge pull request #5820 from vespa-engine/revert-5809-revert-5801-bratseth/lazy-summary-decoding

Revert "Revert "Bratseth/lazy summary decoding""

8 files changed, 907 insertions, 149 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java
index d50006fb82c..e20340a03c0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java
@@ -11,6 +11,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * A docsum definition which knows how to decode a certain class of document
@@ -66,9 +67,15 @@ public class DocsumDefinition {
         return fields.get(fieldIndex);
     }
 
-    /** Returns the index of a field name */
-    public Integer getFieldIndex(String fieldName) {
-        return fieldNameToIndex.get(fieldName);
+    /** Returns the field with this name, or null if none */
+    public DocsumField getField(String fieldName) {
+        Integer index = fieldNameToIndex.get(fieldName);
+        if (index == null) return null;
+        return getField(index);
+    }
+
+    public Set<String> fieldNames() {
+        return fieldNameToIndex.keySet();
     }
 
     @Override
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
index f1f5e2b7403..298bd839393 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.fastsearch;
 
+import com.yahoo.data.access.ObjectTraverser;
 import com.yahoo.document.GlobalId;
 import com.yahoo.fs4.QueryPacketData;
 import com.yahoo.net.URI;
@@ -8,6 +9,17 @@ import com.yahoo.search.result.Hit;
 import com.yahoo.search.result.Relevance;
 import com.yahoo.data.access.Inspector;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
 /**
  * A regular hit from a Vespa backend
  *
@@ -34,6 +46,24 @@ public class FastHit extends Hit {
     private transient CacheKey cacheKey = null;
 
     /**
+     * Summaries added to this hit which are not yet decoded into fields.
+     * Fields are resolved by returning the first non-null value found by
+     * 1) the field value from the Map of fields in the Hit supertype, and
+     * 2) each of the summaries, in the order of the list (which is the add order).
+     * This ensures that values set from code overwrites any value received as
+     * summary data.
+     *
+     * The reason we keep this rather than eagerly decoding into a the field map
+     * is to reduce garbage collection and decoding cost, with the assumption
+     * that most fields passes through the container with no processing most
+     * of the time.
+     */
+    private List<SummaryData> summaries = new ArrayList<>(1);
+
+    /** Removed field values, which should therefore not be returned if present in summary data */
+    private Set<String> removedFields = null;
+
+    /**
      * Creates an empty and temporarily invalid summary hit
      */
     public FastHit() { }
@@ -128,16 +158,9 @@ public class FastHit extends Hit {
 
     /** For internal use */
     public void addSummary(DocsumDefinition docsumDef, Inspector value) {
-        reserve(docsumDef.getFieldCount());
-        for (DocsumField field : docsumDef.getFields()) {
-            String fieldName = field.getName();
-            Inspector f = value.field(fieldName);
-            if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) {
-                if (super.getField(fieldName) == null) {
-                    setField(fieldName, field.convert(f));
-                }
-            }
-        }
+        if (removedFields != null)
+            removedFields.removeAll(docsumDef.fieldNames());
+        summaries.add(new SummaryData(this, docsumDef, value, summaries.size()));
     }
 
     /**
@@ -172,8 +195,132 @@ public class FastHit extends Hit {
      * </ul>
      */
     @Override
-    public Object getField(String key) {
-        return super.getField(key);
+    public Object getField(String name) {
+        Object value = super.getField(name);
+        if (value != null) return value;
+        value = getSummaryValue(name);
+        if (value != null)
+            super.setField(name, value);
+        return value;
+    }
+
+    @Override
+    public Object setField(String name, Object value) {
+        if (removedFields != null) {
+            if (removedFields.remove(name)) {
+                if (removedFields.isEmpty())
+                    removedFields = null;
+            }
+        }
+        Object oldValue = super.setField(name, value);
+        if (oldValue != null) return oldValue;
+        return getSummaryValue(name);
+    }
+
+    /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */
+    @Override
+    public Map<String, Object> fields() {
+        Map<String, Object> fields = new HashMap<>();
+        for (Iterator<Map.Entry<String, Object>> i = fieldIterator(); i.hasNext(); ) {
+            Map.Entry<String, Object> field = i.next();
+            fields.put(field.getKey(), field.getValue());
+        }
+        return fields;
+    }
+
+    /** Returns a modifiable iterator over the fields of this */
+    @Override
+    public Iterator<Map.Entry<String, Object>> fieldIterator() {
+        return new FieldIterator(this, super.fieldIterator());
+    }
+
+    /** Returns a modifiable iterator over the field names of this */
+    Iterator<String> fieldNameIterator() {
+        return new FieldNameIterator(this, super.fieldKeys().iterator());
+    }
+
+    /** Removes all fields of this */
+    @Override
+    public void clearFields() {
+        summaries.clear();
+        if (removedFields != null)
+            removedFields = null;
+        super.clearFields();
+    }
+
+    /**
+     * Removes a field from this
+     *
+     * @return the removed value of the field, or null if none
+     */
+    @Override
+    public Object removeField(String name) {
+        Object removedValue = super.removeField(name);
+        if (removedValue == null)
+            removedValue = getSummaryValue(name);
+
+        if (removedValue != null) {
+            if (removedFields == null)
+                removedFields = new HashSet<>(2);
+            removedFields.add(name);
+        }
+
+        return removedValue;
+    }
+
+    /**
+     * Returns the keys of the fields of this hit as a modifiable view.
+     * This follows the rules of key sets returned from maps: Key removals are reflected
+     * in the map, add and addAll is not supported.
+     */
+    @Override
+    public Set<String> fieldKeys() {
+        return new FieldSet(this);
+    }
+
+    private Set<String> mapFieldKeys() {
+        return super.fieldKeys();
+    }
+
+    /** Returns whether this field is present <b>in the field map in the parent hit</b> */
+    // Note: If this is made public it must be changed to also check the summary data
+    //       (and internal usage must change to another method).
+    @Override
+    protected boolean hasField(String name) {
+        return super.hasField(name);
+    }
+
+    /** Returns whether any fields are present <b>in the field map in the parent hit</b> */
+    // Note: If this is made public it must be changed to also check the summary data
+    //       (and internal usage must change to another method).
+    @Override
+    protected boolean hasFields() {
+        return super.hasFields();
+    }
+
+    /**
+     * Changes the key under which a value is found. This is useful because it allows keys to be changed
+     * without accessing the value (which may be lazily created).
+     *
+     * @deprecated do not use
+     */
+    @Deprecated
+    @Override
+    @SuppressWarnings("deprecation")
+    public void changeFieldKey(String oldKey, String newKey) {
+        Object value = removeField(oldKey);
+        if (value != null)
+            setField(newKey, value);
+    }
+
+    private Object getSummaryValue(String name) {
+        if (removedFields != null && removedFields.contains(name))
+            return null;
+        for (SummaryData summaryData : summaries) {
+            Object value = summaryData.getField(name);
+            if (value != null) return value;
+        }
+        return null;
     }
 
     @Override
@@ -192,6 +339,8 @@ public class FastHit extends Hit {
         }
     }
 
+    /** @deprecated do not use */
+    @Deprecated // TODO: Make private on Vespa 7
     public static String asHexString(GlobalId gid) {
         StringBuilder sb = new StringBuilder();
         byte[] rawGid = gid.getRawId();
@@ -205,4 +354,415 @@ public class FastHit extends Hit {
         return sb.toString();
     }
 
+    /** A set view of all the field names in this hit. Add/addAll is not supported but remove is. */
+    private static class FieldSet implements Set<String> {
+
+        // A set implementation which tries to avoid creating an actual set when possible.
+        // With more work this could be optimized to avoid creating the set in additional cases.
+
+        private final FastHit hit;
+
+        /** The computed set of fields. Lazily created as it is not always needed. */
+        private Set<String> fieldSet = null;
+
+        public FieldSet(FastHit hit) {
+            this.hit = hit;
+        }
+
+        @Override
+        public int size() {
+            return createSet().size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            if ( ! hit.hasFields() && hit.summaries.isEmpty()) return true;
+            return createSet().isEmpty();
+        }
+
+        @Override
+        public boolean contains(Object o) {
+            String field = (String)o;
+            if (hit.hasField(field)) return true;
+            return createSet().contains(field);
+        }
+
+        @Override
+        public Object[] toArray() {
+            return createSet().toArray();
+        }
+
+        @Override
+        public <T> T[] toArray(T[] a) {
+            return createSet().toArray(a);
+        }
+
+        @Override
+        public boolean add(String s) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean remove(Object o) {
+            String field = (String)o;
+            boolean removed = hit.removeField(field) != null;
+            if (fieldSet != null)
+                fieldSet.remove(field);
+            return removed;
+        }
+
+        @Override
+        public boolean containsAll(Collection<?> c) {
+            for (Object field : c)
+                if ( ! contains(field))
+                    return false;
+            return true;
+        }
+
+        @Override
+        public boolean addAll(Collection<? extends String> c) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean retainAll(Collection<?> c) {
+            Set<?> toRetain = c instanceof Set<?> ? (Set<?>)c : new HashSet<Object>(c);
+            boolean anyRemoved = false;
+            for (Iterator<String> i = iterator(); i.hasNext(); ) {
+                String field = i.next();
+                if (toRetain.contains(field)) continue;
+
+                i.remove();
+                anyRemoved = true;
+            }
+            return anyRemoved;
+        }
+
+        @Override
+        public boolean removeAll(Collection<?> c) {
+            boolean anyRemoved = false;
+            for (Object field : c)
+                if (remove(field))
+                    anyRemoved = true;
+            return anyRemoved;
+        }
+
+        @Override
+        public void clear() {
+            hit.clearFields();
+            fieldSet = null;
+        }
+
+        @Override
+        public Iterator<String> iterator() {
+            return hit.fieldNameIterator();
+        }
+
+        @Override
+        public int hashCode() {
+            return createSet().hashCode();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (o == this) return true;
+            if ( ! (o instanceof Set)) return false;
+            return createSet().equals(o);
+        }
+
+        @Override
+        public String toString() {
+            return createSet().toString();
+        }
+
+        private Set<String> createSet() {
+            if (this.fieldSet != null) return this.fieldSet;
+            if ( ! hit.hasFields() && hit.summaries.isEmpty()) return Collections.emptySet(); // shortcut
+
+            Set<String> fields = new HashSet<>();
+            if (hit.hasFields())
+                fields.addAll(hit.mapFieldKeys());
+
+            for (SummaryData summaryData : hit.summaries)
+                summaryData.data.traverse((ObjectTraverser)(name, __) -> fields.add(name));
+            if (hit.removedFields != null)
+                fields.removeAll(hit.removedFields);
+
+            this.fieldSet = fields;
+
+            return fields;
+        }
+
+    }
+
+    /** Summary data (values of a number of fields) received for this hit */
+    private static class SummaryData {
+
+        private final FastHit hit;
+        private final DocsumDefinition type;
+        private final Inspector data;
+
+        /** The index of this summary in the list of summaries added to this */
+        private final int index;
+
+        SummaryData(FastHit hit, DocsumDefinition type, Inspector data, int index) {
+            this.hit = hit;
+            this.type = type;
+            this.data = data;
+            this.index = index;
+        }
+
+        Object getField(String name) {
+            DocsumField fieldType = type.getField(name);
+            if (fieldType == null) return null;
+            Inspector fieldValue = data.field(name);
+            if ( ! fieldValue.valid() && ! fieldType.getEmulConfig().forceFillEmptyFields()) return null;
+            return fieldType.convert(fieldValue);
+        }
+
+        /** Decodes the given summary into the field map in the parent */
+        private void decodeInto(FastHit hit) {
+            hit.reserve(type.getFieldCount());
+            for (DocsumField field : type.getFields()) {
+                String fieldName = field.getName();
+                Inspector f = data.field(fieldName);
+                if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) {
+                    if (hit.getField(fieldName) == null)
+                        hit.setField(fieldName, field.convert(f));
+                }
+            }
+        }
+
+        private Iterator<Map.Entry<String, Object>> fieldIterator() {
+            return new SummaryDataFieldIterator(hit, type, data.fields().iterator(), index);
+        }
+
+        private Iterator<String> fieldNameIterator() {
+            return new SummaryDataFieldNameIterator(hit, data.fields().iterator(), index);
+        }
+
+        /**
+         * Abstract superclass of iterators over SummaryData content which takes care of skipping unknown,
+         * removed and already returned fields. Read only.
+         */
+        private static abstract class SummaryDataIterator<VALUE> implements Iterator<VALUE> {
+
+            private final FastHit hit;
+            private final Iterator<Map.Entry<String, Inspector>> fieldIterator;
+            private final int index;
+
+            /** The next value or null if none, eagerly read because we need to skip removed and overwritten values */
+            private VALUE next;
+
+            SummaryDataIterator(FastHit hit,
+                                Iterator<Map.Entry<String, Inspector>> fieldIterator,
+                                int index) {
+                this.hit = hit;
+                this.fieldIterator = fieldIterator;
+                this.index = index;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return next != null;
+            }
+
+            @Override
+            public VALUE next() {
+                if (next == null) throw new NoSuchElementException();
+
+                VALUE returnValue = next;
+                advanceNext();
+                return returnValue;
+            }
+
+            protected abstract VALUE toValue(Map.Entry<String, Inspector> field);
+
+            protected void advanceNext() {
+                while (fieldIterator.hasNext()) {
+                    Map.Entry<String, Inspector> nextEntry = fieldIterator.next();
+                    String fieldName = nextEntry.getKey();
+                    next = toValue(nextEntry);
+                    if ( next != null &&
+                         ! hit.hasField(fieldName) &&
+                         ! isRemoved(fieldName) &&
+                         ! isAlreadyReturned(fieldName))
+                        return;
+                }
+                next = null;
+            }
+
+            private boolean isRemoved(String fieldName) {
+                return hit.removedFields != null && hit.removedFields.contains(fieldName);
+            }
+
+            private boolean isAlreadyReturned(String fieldName) {
+                for (int i = 0; i < index; i++) {
+                    if (hit.summaries.get(i).type.fieldNames().contains(fieldName))
+                        return true;
+                }
+                return false;
+            }
+
+        }
+
+
+        /** Iterator over the fields in a SummaryData instance. Read only. */
+        private static class SummaryDataFieldIterator extends SummaryDataIterator<Map.Entry<String, Object>> {
+
+            private final DocsumDefinition type;
+
+            SummaryDataFieldIterator(FastHit hit,
+                                     DocsumDefinition type,
+                                     Iterator<Map.Entry<String, Inspector>> fieldIterator,
+                                     int index) {
+                super(hit, fieldIterator, index);
+                this.type = type;
+                advanceNext();
+            }
+
+            @Override
+            protected Map.Entry<String, Object> toValue(Map.Entry<String, Inspector> field) {
+                DocsumField fieldType = type.getField(field.getKey());
+                if (fieldType == null) return null; // type and content mismatch: May happen during reconfig
+                return new SummaryFieldEntry(field.getKey(), fieldType.convert(field.getValue()));
+            }
+
+            private static final class SummaryFieldEntry implements Map.Entry<String, Object> {
+
+                private final String key;
+                private final Object value;
+
+                public SummaryFieldEntry(String key, Object value) {
+                    this.key = key;
+                    this.value = value;
+                }
+
+                @Override
+                public String getKey() { return key; }
+
+                @Override
+                public Object getValue() { return value; }
+
+                @Override
+                public Object setValue(Object value) { throw new UnsupportedOperationException(); }
+
+            }
+
+        }
+
+        /** Iterator over the field names in a SummaryData instance. Read only. */
+        private static class SummaryDataFieldNameIterator extends SummaryDataIterator<String> {
+
+            SummaryDataFieldNameIterator(FastHit hit,
+                                         Iterator<Map.Entry<String, Inspector>> fieldIterator,
+                                         int index) {
+                super(hit, fieldIterator, index);
+                advanceNext();
+            }
+
+            @Override
+            protected String toValue(Map.Entry<String, Inspector> field) { return field.getKey(); }
+
+        }
+    }
+
+    /**
+     * Abstract superclass of iterators over all the field content of a FastHit.
+     * This handles iterating over the iterators of Hit and the SummaryData instances of the FastHit,
+     * to provide a view of all the summary data of the FastHit.
+     * Iteration over fields of each piece of data (of Hit or a SummaryData instance) is delegated to the
+     * iterators of those types.
+     */
+    private static abstract class SummaryIterator<VALUE> implements Iterator<VALUE> {
+
+        private final FastHit hit;
+
+        /** -1 means that the current iterator is the map iterator of the parent hit, not any summary data iterator */
+        private int currentSummaryDataIndex = -1;
+        private Iterator<VALUE> currentIterator;
+        private VALUE previousReturned = null;
+
+        public SummaryIterator(FastHit hit, Iterator<VALUE> mapFieldsIterator) {
+            this.hit = hit;
+            this.currentIterator = mapFieldsIterator;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (currentIterator.hasNext()) return true;
+            return nextIterator();
+        }
+
+        @Override
+        public VALUE next() {
+            if (currentIterator.hasNext() || nextIterator()) return previousReturned = currentIterator.next();
+            throw new NoSuchElementException();
+        }
+
+        @Override
+        public void remove() {
+            if (previousReturned == null)
+                throw new IllegalStateException();
+            if ( ! ( currentIterator instanceof SummaryData.SummaryDataIterator))
+                currentIterator.remove(); // remove from the map
+            if (hit.removedFields == null)
+                hit.removedFields = new HashSet<>();
+            hit.removedFields.add(nameOf(previousReturned));
+            previousReturned = null;
+        }
+
+        protected abstract String nameOf(VALUE value);
+        protected abstract Iterator<VALUE> iteratorFor(SummaryData summary);
+
+        /** Advanced to the next non-empty iterator, if any */
+        private boolean nextIterator() {
+            while (++currentSummaryDataIndex < hit.summaries.size()) {
+                currentIterator = iteratorFor(hit.summaries.get(currentSummaryDataIndex));
+                if (currentIterator.hasNext())
+                    return true;
+            }
+            return false;
+        }
+
+    }
+
+    /** Iterator over all the field content of a FastHit */
+    private static class FieldIterator extends SummaryIterator<Map.Entry<String, Object>> {
+
+        public FieldIterator(FastHit hit, Iterator<Map.Entry<String, Object>> mapFieldsIterator) {
+            super(hit, mapFieldsIterator);
+        }
+
+        @Override
+        protected String nameOf(Map.Entry<String, Object> value) {
+            return value.getKey();
+        }
+
+        @Override
+        protected Iterator<Map.Entry<String, Object>> iteratorFor(SummaryData summary) {
+            return summary.fieldIterator();
+        }
+
+    }
+
+    /** Iterator over all the field names stored in a FastHit */
+    private static class FieldNameIterator extends SummaryIterator<String> {
+
+        public FieldNameIterator(FastHit hit, Iterator<String> mapFieldNamesIterator) {
+            super(hit, mapFieldNamesIterator);
+        }
+
+        @Override
+        protected String nameOf(String value) {
+            return value;
+        }
+
+        @Override
+        protected Iterator<String> iteratorFor(SummaryData summary) {
+            return summary.fieldNameIterator();
+        }
+
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 0fed379d446..f13fb2e88f4 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -151,11 +151,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
 
     //--------------  Generic property containers --------------------------------
 
-    /**
-     * The synchronous view of the JDisc request causing this query.
-     *
-     * @since 5.1
-     */
+    /** The synchronous view of the JDisc request causing this query */
     private final HttpRequest httpRequest;
 
     /** The context, or null if there is no context */
@@ -195,7 +191,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
 
     private static QueryProfileType argumentType;
     static {
-        argumentType=new QueryProfileType("native");
+        argumentType = new QueryProfileType("native");
         argumentType.setBuiltin(true);
 
         argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start"));
@@ -622,7 +618,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
      *
      * @deprecated this is ignored
      */
-    @Deprecated
+    @Deprecated // TODO: Remove on Vespa 7
     public void setCompress(boolean ignored) { }
 
     /**
@@ -630,7 +626,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
      *
      * @deprecated this always returns false
      */
-    @Deprecated
+    @Deprecated // TODO: Remove on Vespa 7
     public boolean getCompress() { return false; }
 
     /** Returns a string describing this query */
diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java
index 0664720c7d7..15c148b7db7 100644
--- a/container-search/src/main/java/com/yahoo/search/result/Hit.java
+++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java
@@ -401,14 +401,9 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
     /** Returns the name of the source creating this hit */
     public String getSource() { return source; }
 
-    /**
-     * Returns the fields of this as a read-only map. This is more costly than the preferred iterator(), as
-     * it uses Collections.unmodifiableMap()
-     *
-     * @return An readonly map of the fields
-     */
+    /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */
     // TODO Should it be deprecated ?
-    public final Map<String, Object> fields() { return getUnmodifiableFieldMap(); }
+    public Map<String, Object> fields() { return getUnmodifiableFieldMap(); }
 
     /** Allocate room for the given number of fields to avoid resizing. */
     public void reserve(int minSize) {
@@ -424,14 +419,10 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
         return getFieldMap().put(key, value);
     }
 
-    /**
-     * Returns an iterator over the fields of this
-     * 
-     * @return an iterator for traversing the fields of this hit
-     */
-    public final Iterator<Map.Entry<String,Object>> fieldIterator() { return getFieldMap().entrySet().iterator(); }
+    /** Returns a modifiable iterator over the fields of this */
+    public Iterator<Map.Entry<String, Object>> fieldIterator() { return getFieldMap().entrySet().iterator(); }
 
-    /** Returns a field value */
+    /** Returns a field value or null if not present */
     public Object getField(String value) { return fields != null ? fields.get(value) : null; }
 
     /** Removes all fields of this */
@@ -457,10 +448,22 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
         return getFieldMap().keySet();
     }
 
+    protected boolean hasField(String name) {
+        return fields != null && fields.containsKey(name);
+    }
+
+    /** Returns whether any fields are set in this */
+    protected boolean hasFields() {
+        return fields != null && ! fields.isEmpty();
+    }
+
     /**
      * Changes the key under which a value is found. This is useful because it allows keys to be changed
      * without accessing the value (which may be lazily created).
+     *
+     * @deprecated do not use
      */
+    @Deprecated // TODO: Remove on Vespa 7
     public void changeFieldKey(String oldKey, String newKey) {
         Map<String,Object> fieldMap = getFieldMap();
         Object value = fieldMap.remove(oldKey);
@@ -468,7 +471,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
     }
 
     private Map<String, Object> getFieldMap() {
-        return getFieldMap(16);
+        return getFieldMap(2);
     }
 
     private Map<String, Object> getFieldMap(int minSize) {
diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
index eb1b3a68219..a20949e3dfd 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
@@ -21,14 +21,14 @@ import com.yahoo.yolean.chain.Provides;
 /**
  * Minimal combinator for YQL+ syntax and heuristically parsed user queries.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- * @since 5.1.28
+ * @author Steinar Knutsen
  */
 @Beta
 @Provides(MinimalQueryInserter.EXTERNAL_YQL)
 @Before(PhaseNames.TRANSFORMED_QUERY)
 @After("com.yahoo.prelude.statistics.StatisticsSearcher")
 public class MinimalQueryInserter extends Searcher {
+
     public static final String EXTERNAL_YQL = "ExternalYql";
 
     public static final CompoundName YQL = new CompoundName("yql");
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
index 9b2de75e272..83871b559b4 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
@@ -1,167 +1,323 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.fastsearch;
 
-
+import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.container.search.LegacyEmulationConfig;
 import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.prelude.hitfield.XMLString;
 import com.yahoo.prelude.hitfield.JSONString;
+import com.yahoo.search.result.Hit;
 import com.yahoo.search.result.NanNumber;
 import com.yahoo.search.result.StructuredData;
-import com.yahoo.slime.*;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 
 import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
 
+import com.yahoo.slime.BinaryFormat;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import org.junit.Test;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.CoreMatchers.*;
+
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
-
+import static org.junit.Assert.assertTrue;
 
 public class SlimeSummaryTestCase {
 
+    private static final String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
+    private static final String partial_summary1_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg";
+    private static final String partial_summary2_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg";
+
     @Test
     public void testDecodingEmpty() {
-        String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
-        LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true));
-        DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul);
-        byte[] docsum = makeEmptyDocsum();
+        DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf);
         FastHit hit = new FastHit();
-        assertNull(set.lazyDecode("default", docsum, hit));
-        assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN));
-        assertThat(hit.getField("short_field"),   equalTo(NanNumber.NaN));
-        assertThat(hit.getField("byte_field"),    equalTo(NanNumber.NaN));
-        assertThat(hit.getField("float_field"),   equalTo(NanNumber.NaN));
-        assertThat(hit.getField("double_field"),  equalTo(NanNumber.NaN));
-        assertThat(hit.getField("int64_field"),   equalTo(NanNumber.NaN));
-        assertThat(hit.getField("string_field"),  equalTo(""));
-        assertThat(hit.getField("data_field"), instanceOf(RawData.class));
-        assertThat(hit.getField("data_field").toString(), equalTo(""));
-        assertThat(hit.getField("longstring_field"), equalTo((Object)""));
-        assertThat(hit.getField("longdata_field"), instanceOf(RawData.class));
-        assertThat(hit.getField("longdata_field").toString(), equalTo(""));
-        assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class));
-        assertThat(hit.getField("xmlstring_field").toString(), equalTo(""));
-        // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class));
-        assertThat(hit.getField("jsonstring_field").toString(), equalTo(""));
-        // Empty tensors are represented by null because we don't have type information here to create the right empty tensor
+        assertNull(docsum.lazyDecode("default", emptySummary(), hit));
+        assertNull(hit.getField("integer_field"));
+        assertNull(hit.getField("short_field"));
+        assertNull(hit.getField("byte_field"));
+        assertNull(hit.getField("float_field"));
+        assertNull(hit.getField("double_field"));
+        assertNull(hit.getField("int64_field"));
+        assertNull(hit.getField("string_field"));
+        assertNull(hit.getField("data_field"));
+        assertNull(hit.getField("data_field"));
+        assertNull(hit.getField("longstring_field"));
+        assertNull(hit.getField("longdata_field"));
+        assertNull(hit.getField("longdata_field"));
+        assertNull(hit.getField("xmlstring_field"));
+        assertNull(hit.getField("xmlstring_field"));
+        assertNull(hit.getField("jsonstring_field"));
         assertNull(hit.getField("tensor_field1"));
         assertNull(hit.getField("tensor_field2"));
     }
 
-    private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
-    }
-
     @Test
-    public void testDecodingEmptyWithoutForcedFill() {
-        String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
-        DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false)));
-        byte[] docsum = makeEmptyDocsum();
+    public void testDecodingEmptyWithLegacyEmulation() {
+        LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true));
+        DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig);
         FastHit hit = new FastHit();
-        assertNull(set.lazyDecode("default", docsum, hit));
-        assertThat(hit.getField("integer_field"), equalTo(null));
-        assertThat(hit.getField("short_field"),   equalTo(null));
-        assertThat(hit.getField("byte_field"),    equalTo(null));
-        assertThat(hit.getField("float_field"),   equalTo(null));
-        assertThat(hit.getField("double_field"),  equalTo(null));
-        assertThat(hit.getField("int64_field"),   equalTo(null));
-        assertThat(hit.getField("string_field"),  equalTo(null));
-        assertThat(hit.getField("data_field"), equalTo(null));
-        assertThat(hit.getField("data_field"), equalTo(null));
-        assertThat(hit.getField("longstring_field"), equalTo(null));
-        assertThat(hit.getField("longdata_field"), equalTo(null));
-        assertThat(hit.getField("longdata_field"), equalTo(null));
-        assertThat(hit.getField("xmlstring_field"), equalTo(null));
-        assertThat(hit.getField("xmlstring_field"), equalTo(null));
-        assertThat(hit.getField("jsonstring_field"), equalTo(null));
+        assertNull(docsum.lazyDecode("default", emptySummary(), hit));
+        assertEquals(NanNumber.NaN, hit.getField("integer_field"));
+        assertEquals(NanNumber.NaN, hit.getField("short_field"));
+        assertEquals(NanNumber.NaN, hit.getField("byte_field"));
+        assertEquals(NanNumber.NaN, hit.getField("float_field"));
+        assertEquals(NanNumber.NaN, hit.getField("double_field"));
+        assertEquals(NanNumber.NaN, hit.getField("int64_field"));
+        assertEquals("", hit.getField("string_field"));
+        assertEquals(RawData.class, hit.getField("data_field").getClass());
+        assertEquals("", hit.getField("data_field").toString());
+        assertEquals("", hit.getField("longstring_field"));
+        assertEquals(RawData.class, hit.getField("longdata_field").getClass());
+        assertEquals("", hit.getField("longdata_field").toString());
+        assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass());
+        assertEquals("", hit.getField("xmlstring_field").toString());
+        // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class));
+        assertEquals("", hit.getField("jsonstring_field").toString());
+        // Empty tensors are represented by null because we don't have type information here to create the right empty tensor
         assertNull(hit.getField("tensor_field1"));
         assertNull(hit.getField("tensor_field2"));
     }
 
-    private byte[] makeEmptyDocsum() {
-        Slime slime = new Slime();
-        Cursor docsum = slime.setObject();
-        return encode((slime));
-    }
-    private byte [] encode(Slime slime) {
-        byte[] tmp = BinaryFormat.encode(slime);
-        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
-        buf.order(ByteOrder.LITTLE_ENDIAN);
-        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
-        buf.order(ByteOrder.BIG_ENDIAN);
-        buf.put(tmp);
-        return buf.array();
-    }
-
     @Test
     public void testTimeout() {
-        String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
-        DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf);
-        byte[] docsum = makeTimeout();
+        DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf);
         FastHit hit = new FastHit();
-        assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....", set.lazyDecode("default", docsum, hit));
+        assertEquals("Hit hit index:null/0/000000000000000000000000 (relevance null) [fasthit, globalid: 0 0 0 0 0 0 0 0 0 0 0 0, partId: 0, distributionkey: 0] failed: Timed out....",
+                     docsum.lazyDecode("default", timeoutSummary(), hit));
     }
 
     @Test
     public void testDecoding() {
         Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}");
         Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}");
-
-        String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
-        DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf);
-        byte[] docsum = makeDocsum(tensor1, tensor2);
+        DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf);
         FastHit hit = new FastHit();
-        assertNull(set.lazyDecode("default", docsum, hit));
-        assertThat(hit.getField("integer_field"), equalTo(4));
-        assertThat(hit.getField("short_field"),   equalTo((short)2));
-        assertThat(hit.getField("byte_field"),    equalTo((byte)1));
-        assertThat(hit.getField("float_field"),   equalTo(4.5f));
-        assertThat(hit.getField("double_field"),  equalTo(8.75));
-        assertThat(hit.getField("int64_field"),   equalTo(8L));
-        assertThat(hit.getField("string_field"),  equalTo("string_value"));
-        assertThat(hit.getField("data_field"), instanceOf(RawData.class));
-        assertThat(hit.getField("data_field").toString(), equalTo("data_value"));
-        assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value"));
-        assertThat(hit.getField("longdata_field"), instanceOf(RawData.class));
-        assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value"));
-        assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class));
-        assertThat(hit.getField("xmlstring_field").toString(), equalTo("<tag>xmlstring_value</tag>"));
+        assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit));
+        assertEquals(4, hit.getField("integer_field"));
+        assertEquals((short)2, hit.getField("short_field"));
+        assertEquals((byte)1, hit.getField("byte_field"));
+        assertEquals(4.5F, hit.getField("float_field"));
+        assertEquals(8.75, hit.getField("double_field"));
+        assertEquals(8L, hit.getField("int64_field"));
+        assertEquals("string_value", hit.getField("string_field"));
+        assertEquals(RawData.class, hit.getField("data_field").getClass());
+        assertEquals("data_value", hit.getField("data_field").toString());
+        assertEquals("longstring_value", hit.getField("longstring_field"));
+        assertEquals(RawData.class, hit.getField("longdata_field").getClass());
+        assertEquals("longdata_value", hit.getField("longdata_field").toString());
+        assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass());
+        assertEquals("<tag>xmlstring_value</tag>", hit.getField("xmlstring_field").toString());
         if (hit.getField("jsonstring_field") instanceof JSONString) {
             JSONString jstr = (JSONString) hit.getField("jsonstring_field");
-            assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}"));
-            assertThat(jstr.getParsedJSON(), notNullValue());
+            assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent());
+            assertNotNull(jstr.getParsedJSON());
 
-            com.yahoo.data.access.Inspectable obj = jstr;
-            com.yahoo.data.access.Inspector value = obj.inspect();
-            assertThat(value.field("foo").asLong(), equalTo(1L));
-            assertThat(value.field("bar").asLong(), equalTo(2L));
+            com.yahoo.data.access.Inspector value = jstr.inspect();
+            assertEquals(1L, value.field("foo").asLong());
+            assertEquals(2L, value.field("bar").asLong());
         } else {
             StructuredData sdata = (StructuredData) hit.getField("jsonstring_field");
-            assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}"));
+            assertEquals("{\"foo\":1,\"bar\":2}", sdata.toJson());
 
-            com.yahoo.data.access.Inspectable obj = sdata;
-            com.yahoo.data.access.Inspector value = obj.inspect();
-            assertThat(value.field("foo").asLong(), equalTo(1L));
-            assertThat(value.field("bar").asLong(), equalTo(2L));
+            com.yahoo.data.access.Inspector value = sdata.inspect();
+            assertEquals(1L, value.field("foo").asLong());
+            assertEquals(2L, value.field("bar").asLong());
         }
         assertEquals(tensor1, hit.getField("tensor_field1"));
         assertEquals(tensor2, hit.getField("tensor_field2"));
     }
 
-    private DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0));
+    @Test
+    public void testFieldAccessAPI() {
+        DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf);
+        DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf);
+        DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf);
+        FastHit hit = new FastHit();
+        Map<String, Object> expected = new HashMap<>();
+
+        assertFields(expected, hit);
+
+        partialDocsum1.lazyDecode("partial1", partialSummary1(), hit);
+        expected.put("integer_field", 4);
+        expected.put("short_field", (short) 2);
+        assertFields(expected, hit);
+
+        partialDocsum2.lazyDecode("partial2", partialSummary2(), hit);
+        expected.put("float_field", 4.5F);
+        expected.put("double_field", 8.75D);
+        assertFields(expected, hit);
+
+        hit.removeField("short_field");
+        expected.remove("short_field");
+        assertFields(expected, hit);
+
+        hit.setField("string", "hello");
+        expected.put("string", "hello");
+        assertFields(expected, hit);
+
+        hit.setField("short_field", 3.8F);
+        expected.put("short_field", 3.8F);
+        assertFields(expected, hit);
+
+        hit.removeField("string");
+        expected.remove("string");
+        assertFields(expected, hit);
+
+        hit.removeField("integer_field");
+        hit.removeField("double_field");
+        expected.remove("integer_field");
+        expected.remove("double_field");
+        assertFields(expected, hit);
+
+        hit.clearFields();
+        expected.clear();
+        assertFields(expected, hit);
+
+        // --- Re-populate
+        partialDocsum1.lazyDecode("partial1", partialSummary1(), hit);
+        expected.put("integer_field", 4);
+        expected.put("short_field", (short) 2);
+        partialDocsum2.lazyDecode("partial2", partialSummary2(), hit);
+        expected.put("float_field", 4.5F);
+        expected.put("double_field", 8.75D);
+        hit.setField("string1", "hello");
+        hit.setField("string2", "hello");
+        expected.put("string1", "hello");
+        expected.put("string2", "hello");
+        assertFields(expected, hit);
+
+        Set<String> keys = hit.fieldKeys();
+        assertTrue(keys.remove("integer_field"));
+        expected.remove("integer_field");
+        assertTrue(keys.remove("string2"));
+        expected.remove("string2");
+        assertFields(expected, hit);
+        assertFalse(keys.remove("notpresent"));
+
+        assertTrue(keys.retainAll(ImmutableSet.of("string1", "notpresent", "double_field")));
+        expected.remove("short_field");
+        expected.remove("float_field");
+        assertFields(expected, hit);
+
+        Iterator<String> keyIterator = keys.iterator();
+        assertEquals("string1", keyIterator.next());
+        keyIterator.remove();
+        expected.remove("string1");
+        assertFields(expected, hit);
+
+        assertEquals("double_field", keyIterator.next());
+        keyIterator.remove();
+        expected.remove("double_field");
+        assertFields(expected, hit);
+
+        // --- Re-populate
+        partialDocsum1.lazyDecode("partial1", partialSummary1(), hit);
+        expected.put("integer_field", 4);
+        expected.put("short_field", (short) 2);
+        partialDocsum2.lazyDecode("partial2", partialSummary2(), hit);
+        expected.put("float_field", 4.5F);
+        expected.put("double_field", 8.75D);
+        hit.setField("string", "hello");
+        expected.put("string", "hello");
+        assertFields(expected, hit);
+
+        Iterator<Map.Entry<String, Object>> fieldIterator = hit.fieldIterator();
+        assertEquals("string", fieldIterator.next().getKey());
+        fieldIterator.remove();
+        expected.remove("string");
+        assertFields(expected, hit);
+
+        fieldIterator.next();
+        assertEquals("short_field", fieldIterator.next().getKey());
+        fieldIterator.remove();
+        expected.remove("short_field");
+        assertFields(expected, hit);
+
+        fieldIterator.next();
+        assertEquals("double_field", fieldIterator.next().getKey());
+        fieldIterator.remove();
+        expected.remove("double_field");
+        assertFields(expected, hit);
+
+        fieldIterator = hit.fieldIterator();
+        assertEquals("float_field", fieldIterator.next().getKey());
+        fieldIterator.remove();
+        expected.remove("float_field");
+        assertFields(expected, hit);
+
+        assertEquals("integer_field", fieldIterator.next().getKey());
+        fieldIterator.remove();
+        expected.remove("integer_field");
+        assertFields(expected, hit);
     }
 
-    private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) {
+
+    /** Asserts that the expected fields are what is returned from every access method of Hit */
+    private void assertFields(Map<String, Object> expected, Hit hit) {
+        // fieldKeys
+        int fieldNameIteratorFieldCount = 0;
+        for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) {
+            fieldNameIteratorFieldCount++;
+            assertTrue(expected.containsKey(i.next()));
+        }
+        assertEquals(expected.size(), fieldNameIteratorFieldCount);
+        assertEquals(expected.keySet(), hit.fieldKeys());
+        // getField
+        for (Map.Entry<String, Object> field : expected.entrySet())
+            assertEquals(field.getValue(), hit.getField(field.getKey()));
+        // fields
+        assertEquals(expected, hit.fields());
+        // fieldIterator
+        int fieldIteratorFieldCount = 0;
+        for (Iterator<Map.Entry<String, Object>> i = hit.fieldIterator(); i.hasNext(); ) {
+            fieldIteratorFieldCount++;
+            Map.Entry<String, Object> field = i.next();
+            assertEquals(field.getValue(), expected.get(field.getKey()));
+        }
+        assertEquals(expected.size(), fieldIteratorFieldCount);
+    }
+
+    private byte[] emptySummary() {
+        Slime slime = new Slime();
+        slime.setObject();
+        return encode((slime));
+    }
+
+    private byte [] timeoutSummary() {
+        Slime slime = new Slime();
+        slime.setString("Timed out....");
+        return encode((slime));
+    }
+
+    private byte[] partialSummary1() {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        docsum.setLong("integer_field", 4);
+        docsum.setLong("short_field", 2);
+        return encode((slime));
+    }
+
+    private byte[] partialSummary2() {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        docsum.setLong("integer_field", 4);
+        docsum.setDouble("float_field", 4.5);
+        docsum.setDouble("double_field", 8.75);
+        return encode((slime));
+    }
+
+    private byte[] fullSummary(Tensor tensor1, Tensor tensor2) {
         Slime slime = new Slime();
         Cursor docsum = slime.setObject();
         docsum.setLong("integer_field", 4);
@@ -185,10 +341,24 @@ public class SlimeSummaryTestCase {
         return encode((slime));
     }
 
-    private byte [] makeTimeout() {
-        Slime slime = new Slime();
-        slime.setString("Timed out....");
-        return encode((slime));
+    private byte[] encode(Slime slime) {
+        byte[] tmp = BinaryFormat.encode(slime);
+        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
+        buf.order(ByteOrder.LITTLE_ENDIAN);
+        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
+        buf.order(ByteOrder.BIG_ENDIAN);
+        buf.put(tmp);
+        return buf.array();
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0));
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg
new file mode 100644
index 00000000000..5aa5c84d936
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary1.cfg
@@ -0,0 +1,10 @@
+documentdb[1]
+documentdb[0].name test
+documentdb[0].summaryclass[1]
+documentdb[0].summaryclass[0].name default
+documentdb[0].summaryclass[0].id 1
+documentdb[0].summaryclass[0].fields[2]
+documentdb[0].summaryclass[0].fields[0].name integer_field
+documentdb[0].summaryclass[0].fields[0].type integer
+documentdb[0].summaryclass[0].fields[1].name short_field
+documentdb[0].summaryclass[0].fields[1].type short
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg
new file mode 100644
index 00000000000..bc870a63d66
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/partial-summary2.cfg
@@ -0,0 +1,12 @@
+documentdb[1]
+documentdb[0].name test
+documentdb[0].summaryclass[1]
+documentdb[0].summaryclass[0].name default
+documentdb[0].summaryclass[0].id 2
+documentdb[0].summaryclass[0].fields[3]
+documentdb[0].summaryclass[0].fields[0].name integer_field
+documentdb[0].summaryclass[0].fields[0].type integer
+documentdb[0].summaryclass[0].fields[1].name float_field
+documentdb[0].summaryclass[0].fields[1].type float
+documentdb[0].summaryclass[0].fields[2].name double_field
+documentdb[0].summaryclass[0].fields[2].type double