Merge branch 'master' into balder/prepare-remove-visitor-ordering

17 files changed, 185 insertions, 151 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index e69303c2d01..de811cc3053 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -5230,6 +5230,7 @@
       "public void <init>(java.lang.String)",
       "public java.lang.String getName()",
       "public void setName(java.lang.String)",
+      "public java.lang.String toSerialForm()",
       "public java.lang.String toString()",
       "public int hashCode()",
       "public boolean equals(java.lang.Object)",
@@ -5270,7 +5271,7 @@
     ],
     "methods": [
       "public void <init>(java.lang.String)",
-      "public java.lang.String toString()",
+      "public java.lang.String toSerialForm()",
       "public int hashCode()",
       "public boolean equals(java.lang.Object)",
       "public int compare(java.lang.Comparable, java.lang.Comparable)"
@@ -5342,7 +5343,7 @@
       "public com.yahoo.search.query.Sorting$UcaSorter$Strength getStrength()",
       "public com.ibm.icu.text.Collator getCollator()",
       "public java.lang.String getDecomposition()",
-      "public java.lang.String toString()",
+      "public java.lang.String toSerialForm()",
       "public int hashCode()",
       "public boolean equals(java.lang.Object)",
       "public com.yahoo.search.query.Sorting$UcaSorter clone()",
diff --git a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
index 7dabb52a233..941bce67d16 100644
--- a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
+++ b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
@@ -64,12 +64,6 @@ public class GetDocSumsPacket extends Packet {
     public static final int GDF_PROPERTIES = 0x00001000;
     public static final int GDF_FLAGS = 0x00002000;
 
-    /**
-     * flag bits, as given in fastserver4/src/network/transport.h
-     * definition of enum getdocsums_flags
-     */
-    public static final int GDFLAG_IGNORE_ROW  = 0x00000001;
-
     public void encodeBody(ByteBuffer buffer) {
         setFieldsFromHits();
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4FillInvoker.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4FillInvoker.java
index 469ef4864bf..97a04b9853c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4FillInvoker.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4FillInvoker.java
@@ -31,15 +31,8 @@ public class FS4FillInvoker extends FillInvoker {
 
     private int expectedFillResults = 0;
 
-    public FS4FillInvoker(VespaBackEndSearcher searcher, Query query, FS4ResourcePool fs4ResourcePool, String hostname, int port) {
-        this.searcher = searcher;
-        Backend backend = fs4ResourcePool.getBackend(hostname, port);
-        this.channel = backend.openChannel();
-        channel.setQuery(query);
-    }
-
     // fdispatch code path
-    public FS4FillInvoker(VespaBackEndSearcher searcher, Query query, Backend backend) {
+    FS4FillInvoker(VespaBackEndSearcher searcher, Query query, Backend backend) {
         this.searcher = searcher;
         this.channel = backend.openChannel();
         channel.setQuery(query);
@@ -152,7 +145,7 @@ public class FS4FillInvoker extends FillInvoker {
 
         boolean summaryNeedsQuery = searcher.summaryNeedsQuery(result.getQuery());
         if (result.getQuery().getTraceLevel() >= 3)
-            result.getQuery().trace((summaryNeedsQuery ? "Resending " : "Not resending ") + "query during document summary fetching", 3);
+            result.getQuery().trace((summaryNeedsQuery ? "FS4: Resending " : "Not resending ") + "query during document summary fetching", 3);
 
         GetDocSumsPacket docsumsPacket = GetDocSumsPacket.create(result, summaryClass, summaryNeedsQuery);
         int compressionLimit = result.getQuery().properties().getInteger(FS4SearchInvoker.PACKET_COMPRESSION_LIMIT, 0);
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java
index d2c19339298..4a13d24d953 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java
@@ -50,6 +50,7 @@ public class SameElementItem extends NonReducibleCompositeItem {
     @Override
     protected void adding(Item item) {
         super.adding(item);
+        //TODO See if we can require only SimpleIndexedItem instead of TermItem
         Validator.ensureInstanceOf("Child item", item, TermItem.class);
         TermItem asTerm = (TermItem) item;
         Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName());
@@ -59,7 +60,7 @@ public class SameElementItem extends NonReducibleCompositeItem {
     @Override
     public Optional<Item> extractSingleChild() {
         if (getItemCount() == 1) {
-            WordItem child = (WordItem) getItem(0);
+            SimpleIndexedItem child = (SimpleIndexedItem)getItem(0);
             child.setIndexName(getFieldName() + "." + child.getIndexName());
             return Optional.of(child);
         }
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 7105ba8c7ad..8354ed12fb3 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -13,6 +13,8 @@ import com.yahoo.prelude.query.Highlight;
 import com.yahoo.prelude.query.QueryException;
 import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation;
 import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.dispatch.Dispatcher;
+import com.yahoo.search.dispatch.rpc.ProtobufSerialization;
 import com.yahoo.search.federation.FederationSearcher;
 import com.yahoo.search.query.Model;
 import com.yahoo.search.query.ParameterParser;
@@ -102,7 +104,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         WEB(4,"web"),
         PROGRAMMATIC(5, "prog"),
         YQL(6, "yql"),
-        SELECT(7, "select");;
+        SELECT(7, "select");
 
         private final int intValue;
         private final String stringValue;
@@ -219,6 +221,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType())));
         argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType())));
         argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType())));
+        argumentType.addField(new FieldDescription(Dispatcher.DISPATCH, new QueryProfileFieldType(Dispatcher.getArgumentType())));
         argumentType.freeze();
     }
     public static QueryProfileType getArgumentType() { return argumentType; }
@@ -495,7 +498,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     private void appendQueryProfileProperties(CompiledQueryProfile profile,Set<String> mentioned,StringBuilder b) {
         for (Map.Entry<String,Object> property : profile.listValues("", requestProperties()).entrySet()) {
             if ( ! mentioned.contains(property.getKey()))
-                b.append(property.getKey() + "=" + property.getValue() + " (value from query profile)<br/>\n");
+                b.append(property.getKey()).append("=").append(property.getValue()).append(" (value from query profile)<br/>\n");
         }
     }
 
@@ -741,7 +744,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
 
         StringBuilder concatenated = new StringBuilder();
         for (Object message : messages)
-            concatenated.append(String.valueOf(message));
+            concatenated.append(message);
         trace(concatenated.toString(), includeQuery, traceLevel);
     }
 
@@ -1090,25 +1093,14 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
             return Collections.singletonMap("grouping", true);
         if (ranking.getQueryCache())
             return Collections.singletonMap("query", true);
-        return Collections.<String,Boolean>emptyMap();
-    }
-
-    private int computeTraceLevelForBackend() {
-        int traceLevel = getTraceLevel();
-        if (model.getExecution().trace().getForceTimestamps()) {
-            traceLevel = Math.max(traceLevel, 5); // Backend produces timing information on level 4 and 5
-        }
-        if (getExplainLevel() > 0) {
-            traceLevel = Math.max(traceLevel, getExplainLevel() + 5);
-        }
-        return traceLevel;
+        return Collections.emptyMap();
     }
 
     private Map<String, String> createModelMap() {
         Map<String, String> m = new HashMap<>();
         if (model.getSearchPath() != null) m.put("searchpath", model.getSearchPath());
 
-        int traceLevel = computeTraceLevelForBackend();
+        int traceLevel = ProtobufSerialization.getTraceLevelForBackend(this);
         if (traceLevel > 0) m.put("tracelevel", String.valueOf(traceLevel));
 
         return m;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index fb587e394d9..a5c6f3650e0 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -17,6 +17,9 @@ import com.yahoo.search.dispatch.searchcluster.Group;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.PingFactory;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.vespa.config.search.DispatchConfig;
 
@@ -40,16 +43,21 @@ import java.util.Set;
  * @author ollvir
  */
 public class Dispatcher extends AbstractComponent {
+
+    public static final String DISPATCH = "dispatch";
+    private static final String INTERNAL = "internal";
+    private static final String PROTOBUF = "protobuf";
+
     private static final String FDISPATCH_METRIC = "dispatch_fdispatch";
     private static final String INTERNAL_METRIC = "dispatch_internal";
 
     private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3;
 
     /** If enabled, this internal dispatcher will be preferred over fdispatch whenever possible */
-    public static final CompoundName dispatchInternal = new CompoundName("dispatch.internal");
+    public static final CompoundName dispatchInternal = CompoundName.fromComponents(DISPATCH, INTERNAL);
 
     /** If enabled, search queries will use protobuf rpc */
-    public static final CompoundName dispatchProtobuf = new CompoundName("dispatch.protobuf");
+    public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF);
 
     /** A model of the search cluster this dispatches to */
     private final SearchCluster searchCluster;
@@ -63,8 +71,25 @@ public class Dispatcher extends AbstractComponent {
     private final Metric metric;
     private final Metric.Context metricContext;
 
-    public static Dispatcher create(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize,
-            VipStatus vipStatus, Metric metric) {
+    private static final QueryProfileType argumentType;
+
+    static {
+        argumentType = new QueryProfileType(DISPATCH);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType));
+        argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType));
+        argumentType.freeze();
+    }
+
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    public static Dispatcher create(String clusterId,
+                                    DispatchConfig dispatchConfig,
+                                    FS4ResourcePool fs4ResourcePool,
+                                    int containerClusterSize,
+                                    VipStatus vipStatus,
+                                    Metric metric) {
         var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus);
         var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, !dispatchConfig.useFdispatchByDefault());
         var pingFactory = dispatchConfig.useFdispatchByDefault()? new FS4PingFactory(fs4ResourcePool) : rpcFactory;
@@ -72,11 +97,14 @@ public class Dispatcher extends AbstractComponent {
         return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, pingFactory, metric);
     }
 
-    public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, PingFactory pingFactory,
-            Metric metric) {
+    public Dispatcher(SearchCluster searchCluster,
+                      DispatchConfig dispatchConfig,
+                      InvokerFactory invokerFactory,
+                      PingFactory pingFactory,
+                      Metric metric) {
         this.searchCluster = searchCluster;
         this.loadBalancer = new LoadBalancer(searchCluster,
-                dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
+                                  dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
         this.multilevelDispatch = dispatchConfig.useMultilevelDispatch();
         this.internalDispatchByDefault = !dispatchConfig.useFdispatchByDefault();
         this.invokerFactory = invokerFactory;
@@ -105,14 +133,14 @@ public class Dispatcher extends AbstractComponent {
     }
 
     public Optional<SearchInvoker> getSearchInvoker(Query query, VespaBackEndSearcher searcher) {
-        if (multilevelDispatch || !query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) {
+        if (multilevelDispatch || ! query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) {
             emitDispatchMetric(Optional.empty());
             return Optional.empty();
         }
 
         Optional<SearchInvoker> invoker = getSearchPathInvoker(query, searcher);
 
-        if (!invoker.isPresent()) {
+        if (invoker.isEmpty()) {
             invoker = getInternalInvoker(query, searcher);
         }
         if (invoker.isPresent() && query.properties().getBoolean(com.yahoo.search.query.Model.ESTIMATE)) {
@@ -128,17 +156,14 @@ public class Dispatcher extends AbstractComponent {
     // build invoker based on searchpath
     private Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackEndSearcher searcher) {
         String searchPath = query.getModel().getSearchPath();
-        if (searchPath == null) {
-            return Optional.empty();
-        }
+        if (searchPath == null) return Optional.empty();
+
         try {
             List<Node> nodes = SearchPath.selectNodes(searchPath, searchCluster);
-            if (nodes.isEmpty()) {
-                return Optional.empty();
-            } else {
-                query.trace(false, 2, "Dispatching internally with search path ", searchPath);
-                return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), nodes, true);
-            }
+            if (nodes.isEmpty()) return Optional.empty();
+
+            query.trace(false, 2, "Dispatching internally with search path ", searchPath);
+            return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), nodes, true);
         } catch (InvalidSearchPathException e) {
             return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage())));
         }
@@ -158,14 +183,15 @@ public class Dispatcher extends AbstractComponent {
         Set<Integer> rejected = null;
         for (int i = 0; i < max; i++) {
             Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected);
-            if (!groupInCluster.isPresent()) {
-                // No groups available
-                break;
-            }
+            if (groupInCluster.isEmpty()) break; // No groups available
+
             Group group = groupInCluster.get();
             boolean acceptIncompleteCoverage = (i == max - 1);
-            Optional<SearchInvoker> invoker = invokerFactory.createSearchInvoker(searcher, query, OptionalInt.of(group.id()), group.nodes(),
-                    acceptIncompleteCoverage);
+            Optional<SearchInvoker> invoker = invokerFactory.createSearchInvoker(searcher,
+                                                                                 query,
+                                                                                 OptionalInt.of(group.id()),
+                                                                                 group.nodes(),
+                                                                                 acceptIncompleteCoverage);
             if (invoker.isPresent()) {
                 query.trace(false, 2, "Dispatching internally to search group ", group.id());
                 query.getModel().setSearchPath("/" + group.id());
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedFillInvoker.java
deleted file mode 100644
index 644e6f17bdb..00000000000
--- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedFillInvoker.java
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.dispatch;
-
-import com.yahoo.prelude.fastsearch.FastHit;
-import com.yahoo.search.Result;
-import com.yahoo.search.result.Hit;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * InterleavedFillInvoker uses multiple {@link FillInvoker} objects to interface with content
- * nodes in parallel. Operationally it first sends requests with all contained invokers and then
- * collects the results.
- *
- * @author ollivir
- */
-public class InterleavedFillInvoker extends FillInvoker {
-    private final Map<Integer, FillInvoker> invokers;
-    private Map<Integer, Result> expectedFillResults = null;
-
-    public InterleavedFillInvoker(Map<Integer, FillInvoker> invokers) {
-        this.invokers = invokers;
-    }
-
-    @Override
-    protected void sendFillRequest(Result result, String summaryClass) {
-        expectedFillResults = new HashMap<>();
-
-        for (Iterator<Hit> it = result.hits().deepIterator(); it.hasNext();) {
-            Hit hit = it.next();
-            if (hit instanceof FastHit) {
-                FastHit fhit = (FastHit) hit;
-                Result res = expectedFillResults.computeIfAbsent(fhit.getDistributionKey(), dk -> new Result(result.getQuery()));
-                res.hits().add(fhit);
-            }
-        }
-        expectedFillResults.forEach((distKey, partialResult) -> {
-            FillInvoker invoker = invokers.get(distKey);
-            if (invoker != null) {
-                invoker.sendFillRequest(partialResult, summaryClass);
-            }
-        });
-    }
-
-    @Override
-    protected void getFillResults(Result result, String summaryClass) {
-        if (expectedFillResults == null) {
-            return;
-        }
-        expectedFillResults.forEach((distKey, partialResult) -> {
-            FillInvoker invoker = invokers.get(distKey);
-            if (invoker != null) {
-                invoker.getFillResults(partialResult, summaryClass);
-            }
-        });
-    }
-
-    @Override
-    protected void release() {
-        if (!invokers.isEmpty()) {
-            invokers.values().forEach(FillInvoker::close);
-            invokers.clear();
-        }
-    }
-}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
index 13041aa4035..3db5e291a45 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
@@ -72,11 +72,24 @@ public class ProtobufSerialization {
             builder.setCacheGrouping(true);
         }
 
+        builder.setTraceLevel(getTraceLevelForBackend(query));
+
         mergeToSearchRequestFromRanking(query.getRanking(), builder);
 
         return builder.build();
     }
 
+    public static int getTraceLevelForBackend(Query query) {
+        int traceLevel = query.getTraceLevel();
+        if (query.getModel().getExecution().trace().getForceTimestamps()) {
+            traceLevel = Math.max(traceLevel, 5); // Backend produces timing information on level 4 and 5
+        }
+        if (query.getExplainLevel() > 0) {
+            traceLevel = Math.max(traceLevel, query.getExplainLevel() + 5);
+        }
+        return traceLevel;
+    }
+
     private static void mergeToSearchRequestFromRanking(Ranking ranking, SearchProtocol.SearchRequest.Builder builder) {
         builder.setRankProfile(ranking.getProfile());
 
@@ -99,7 +112,7 @@ public class ProtobufSerialization {
     private static void mergeToSearchRequestFromSorting(Sorting sorting, SearchProtocol.SearchRequest.Builder builder) {
         for (var field : sorting.fieldOrders()) {
             var sortField = SearchProtocol.SortField.newBuilder()
-                    .setField(field.getSorter().getName())
+                    .setField(field.getSorter().toSerialForm())
                     .setAscending(field.getSortOrder() == Order.ASCENDING).build();
             builder.addSorting(sortField);
         }
@@ -127,11 +140,17 @@ public class ProtobufSerialization {
             builder.setCacheQuery(true);
             builder.setSessionKey(query.getSessionId(serverId).toString());
         }
-        builder.setRankProfile(query.getRanking().getProfile());
+        builder.setRankProfile(ranking.getProfile());
 
+        if (ranking.getLocation() != null) {
+            builder.setGeoLocation(ranking.getLocation().toString());
+        }
         if (includeQueryData) {
             mergeQueryDataToDocsumRequest(query, builder);
         }
+        if (query.getTraceLevel() >= 3) {
+            query.trace((includeQueryData ? "ProtoBuf: Resending " : "Not resending ") + "query during document summary fetching", 3);
+        }
 
         return builder;
     }
@@ -149,9 +168,7 @@ public class ProtobufSerialization {
         var featureMap = ranking.getFeatures().asMap();
 
         builder.setQueryTreeBlob(serializeQueryTree(query.getModel().getQueryTree()));
-        if (ranking.getLocation() != null) {
-            builder.setGeoLocation(ranking.getLocation().toString());
-        }
+
         MapConverter.convertMapPrimitives(featureMap, builder::addFeatureOverrides);
         MapConverter.convertMapTensors(featureMap, builder::addTensorFeatureOverrides);
         if (query.getPresentation().getHighlight() != null) {
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java
index aa72823c809..712dca2c34a 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java
@@ -7,6 +7,7 @@ import com.yahoo.compress.Compressor;
 import com.yahoo.container.protect.Error;
 import com.yahoo.data.access.Inspector;
 import com.yahoo.data.access.slime.SlimeAdapter;
+import com.yahoo.prelude.Location;
 import com.yahoo.prelude.fastsearch.DocumentDatabase;
 import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.fastsearch.TimeoutException;
@@ -42,10 +43,8 @@ public class RpcFillInvoker extends FillInvoker {
 
     private final DocumentDatabase documentDb;
     private final RpcResourcePool resourcePool;
-
     private GetDocsumsResponseReceiver responseReceiver;
 
-
     RpcFillInvoker(RpcResourcePool resourcePool, DocumentDatabase documentDb) {
         this.documentDb = documentDb;
         this.resourcePool = resourcePool;
@@ -54,12 +53,15 @@ public class RpcFillInvoker extends FillInvoker {
     @Override
     protected void sendFillRequest(Result result, String summaryClass) {
         ListMap<Integer, FastHit> hitsByNode = hitsByNode(result);
+        Query query = result.getQuery();
 
         CompressionType compression = CompressionType
-                .valueOf(result.getQuery().properties().getString(RpcResourcePool.dispatchCompression, "LZ4").toUpperCase());
+                .valueOf(query.properties().getString(RpcResourcePool.dispatchCompression, "LZ4").toUpperCase());
 
-        if (result.getQuery().getTraceLevel() >= 3)
-            result.getQuery().trace("Sending " + hitsByNode.size() + " summary fetch RPC requests", 3);
+        if (query.getTraceLevel() >= 3) {
+            query.trace("Sending " + hitsByNode.size() + " summary fetch RPC requests", 3);
+            query.trace("RpcSlime: Not resending query during document summary fetching", 3);
+        }
 
         responseReceiver = new GetDocsumsResponseReceiver(hitsByNode.size(), resourcePool.compressor(), result);
         for (Map.Entry<Integer, List<FastHit>> nodeHits : hitsByNode.entrySet()) {
@@ -112,13 +114,14 @@ public class RpcFillInvoker extends FillInvoker {
         Query query = result.getQuery();
         String rankProfile = query.getRanking().getProfile();
         byte[] serializedSlime = BinaryFormat
-                .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(), hits));
+                .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(),
+                                query.getSessionId(), query.getRanking().getLocation(), hits));
         double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0;
         Compressor.Compression compressionResult = resourcePool.compress(query, serializedSlime);
         node.getDocsums(hits, compressionResult.type(), serializedSlime.length, compressionResult.data(), responseReceiver, timeoutSeconds);
     }
 
-    static private Slime toSlime(String rankProfile, String summaryClass, String docType, SessionId sessionId, List<FastHit> hits) {
+    static private Slime toSlime(String rankProfile, String summaryClass, String docType, SessionId sessionId, Location location, List<FastHit> hits) {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
         if (summaryClass != null) {
@@ -133,6 +136,9 @@ public class RpcFillInvoker extends FillInvoker {
         if (rankProfile != null) {
             root.setString("ranking", rankProfile);
         }
+        if (location != null) {
+            root.setString("location", location.toString());
+        }
         Cursor gids = root.setArray("gids");
         for (FastHit hit : hits) {
             gids.addData(hit.getGlobalId().getRawId());
@@ -239,7 +245,7 @@ public class RpcFillInvoker extends FillInvoker {
             int skippedHits = 0;
             for (int i = 0; i < hits.size(); i++) {
                 Inspector summary = summaries.entry(i).field("docsum");
-                if (summary.fieldCount() != 0) {
+                if (summary.valid()) {
                     hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName());
                     hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary);
                     hits.get(i).setFilled(summaryClass);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 0cd646914a1..6146751f35f 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -47,13 +47,9 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
         boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query);
 
         if(query.properties().getBoolean(Dispatcher.dispatchProtobuf, dispatchWithProtobuf)) {
-            return Optional.of(new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(),
-                    summaryNeedsQuery));
+            return Optional.of(new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery));
         }
-        if (query.properties().getBoolean(dispatchSummaries, true)
-                && ! summaryNeedsQuery
-                && query.getRanking().getLocation() == null)
-        {
+        if (query.properties().getBoolean(dispatchSummaries, true) && ! summaryNeedsQuery) {
             return Optional.of(new RpcFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query)));
         } else {
             return Optional.empty();
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
index cd4ba191a7d..341b9b2bce3 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
@@ -201,7 +201,7 @@ public class RpcProtobufFillInvoker extends FillInvoker {
             int skippedHits = 0;
             for (int i = 0; i < hits.size(); i++) {
                 Inspector summary = summaries.entry(i).field("docsum");
-                if (summary.fieldCount() != 0) {
+                if (summary.valid()) {
                     hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName());
                     hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary);
                     hits.get(i).setFilled(summaryClass);
diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java
index 5ed58a74627..f06aab09a3d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Model.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Model.java
@@ -34,6 +34,7 @@ import static com.yahoo.text.Lowercase.toLowerCase;
  * @author bratseth
  */
 public class Model implements Cloneable {
+
     /** The type representing the property arguments consumed by this */
     private static final QueryProfileType argumentType;
     private static final CompoundName argumentTypeName;
@@ -67,7 +68,7 @@ public class Model implements Cloneable {
         argumentType.addField(new FieldDescription(SEARCH_PATH, "string", "searchpath"));
         argumentType.addField(new FieldDescription(RESTRICT, "string", "restrict"));
         argumentType.freeze();
-        argumentTypeName=new CompoundName(argumentType.getId().getName());
+        argumentTypeName = new CompoundName(argumentType.getId().getName());
     }
 
     public static QueryProfileType getArgumentType() { return argumentType; }
diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java
index 0edcf408f53..6518dcd5b6d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java
@@ -176,7 +176,7 @@ public class Sorting implements Cloneable {
                 buffer.put((byte) '-');
             }
             usedBytes++;
-            nameBuffer = Utf8.toBytes(fieldOrder.getSorter().toString());
+            nameBuffer = Utf8.toBytes(fieldOrder.getSorter().toSerialForm());
             buffer.put(nameBuffer);
             usedBytes += nameBuffer.length;
             // If this isn't the last element, append a separating space
@@ -203,8 +203,11 @@ public class Sorting implements Cloneable {
         public String getName() { return fieldName; }
         public void setName(String fieldName) { this.fieldName = fieldName; }
 
+        /** Returns the serial form of this which contains all information needed to reconstruct this sorter */
+        public String toSerialForm() { return fieldName; }
+
         @Override
-        public String toString() { return fieldName; }
+        public String toString() { return toSerialForm(); }
 
         @Override
         public int hashCode() { return fieldName.hashCode(); }
@@ -253,7 +256,7 @@ public class Sorting implements Cloneable {
         public LowerCaseSorter(String fieldName) { super(fieldName); }
 
         @Override
-        public String toString() { return "lowercase(" + getName() + ')'; }
+        public String toSerialForm() { return "lowercase(" + getName() + ')'; }
 
         @Override
         public int hashCode() { return 1 + 3*super.hashCode(); }
@@ -323,7 +326,7 @@ public class Sorting implements Cloneable {
         public String getDecomposition() { return (collator.getDecomposition() == Collator.CANONICAL_DECOMPOSITION) ? "CANONICAL_DECOMPOSITION" : "NO_DECOMPOSITION"; }
 
         @Override
-        public String toString() { return "uca(" + getName() + ',' + locale + ',' + ((strength != Strength.UNDEFINED) ? strength.toString() : "PRIMARY") + ')'; }
+        public String toSerialForm() { return "uca(" + getName() + ',' + locale + ',' + ((strength != Strength.UNDEFINED) ? strength.toString() : "PRIMARY") + ')'; }
 
         @Override
         public int hashCode() { return 1 + 3*locale.hashCode() + 5*strength.hashCode() + 7*super.hashCode(); }
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
index e8d8ecb02bb..2f4804f0c48 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -41,7 +41,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
     }
 
     public FieldDescription(String name, String type, String aliases) {
-        this(name,type,aliases,false,true);
+        this(name, type, aliases, false, true);
     }
 
     public FieldDescription(String name, FieldType type, String aliases) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java
index ae9960f6c7c..1382c106ae3 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java
@@ -2,11 +2,17 @@
 package com.yahoo.prelude.query.test;
 
 import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.SameElementItem;
+import com.yahoo.prelude.query.TermItem;
 import com.yahoo.prelude.query.WordItem;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class SameElementItemTestCase {
@@ -85,4 +91,27 @@ public class SameElementItemTestCase {
         }
     }
 
+    private void verifyExtractSingle(TermItem term) {
+        String subFieldName = term.getIndexName();
+        SameElementItem s = new SameElementItem("structa");
+        s.addItem(term);
+        Optional<Item> single =s.extractSingleChild();
+        assertTrue(single.isPresent());
+        assertEquals(((TermItem)single.get()).getIndexName(), s.getFieldName() + "." + subFieldName);
+    }
+
+    @Test
+    public void requireExtractSingleItemToExtractSingles() {
+        verifyExtractSingle(new WordItem("b", "f1"));
+        verifyExtractSingle(new IntItem("7", "f1"));
+    }
+
+    @Test
+    public void requireExtractSingleItemToExtractSinglesOnly() {
+        SameElementItem s = new SameElementItem("structa");
+        s.addItem(new WordItem("b", "f1"));
+        s.addItem(new WordItem("c", "f2"));
+        assertTrue(s.extractSingleChild().isEmpty());
+    }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java
index 6d1f19eeaf2..cc32bfe1572 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java
@@ -105,6 +105,46 @@ public class FillTestCase {
         assertEquals(3L, result.hits().get("hit:3").getField("field2"));
         assertNull(result.hits().get("hit:4").getField("field2"));
 
+        assertNull(result.hits().getError());
+    }
+
+    @Test
+    public void testMissingHits() {
+        Map<Integer, Client.NodeConnection> nodes = new HashMap<>();
+        nodes.put(0, client.createConnection("host0", 123));
+        nodes.put(1, client.createConnection("host1", 123));
+        nodes.put(2, client.createConnection("host2", 123));
+        RpcResourcePool rpcResourcePool = new RpcResourcePool(nodes);
+        RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true);
+
+        Query query = new Query();
+        Result result = new Result(query);
+        result.hits().add(createHit(0, 0));
+        result.hits().add(createHit(2, 1));
+        result.hits().add(createHit(1, 2));
+        result.hits().add(createHit(2, 3));
+        result.hits().add(createHit(0, 4));
+
+        client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0));
+        client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1));
+        client.setDocsumReponse("host1", 2, "summaryClass1", null);
+        client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3));
+        client.setDocsumReponse("host0", 4, "summaryClass1", null);
+
+        factory.createFillInvoker(db()).fill(result, "summaryClass1");
+
+        assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString());
+        assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString());
+        assertNull(result.hits().get("hit:2").getField("field1"));
+        assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString());
+        assertNull(result.hits().get("hit:4").getField("field1"));
+
+        assertEquals(0L, result.hits().get("hit:0").getField("field2"));
+        assertEquals(1L, result.hits().get("hit:1").getField("field2"));
+        assertNull(result.hits().get("hit:2").getField("field2"));
+        assertEquals(3L, result.hits().get("hit:3").getField("field2"));
+        assertNull(result.hits().get("hit:4").getField("field2"));
+
         assertEquals("Missing hit summary data for summary summaryClass1 for 2 hits", result.hits().getError().getDetailedMessage());
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java
index b6b7a1f5819..2fc8c0fd620 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java
@@ -71,6 +71,8 @@ public class MockClient implements Client {
             Cursor root = responseSlime.setObject();
             Cursor docsums = root.setArray("docsums");
             for (Map<String, Object> docsumFields : docsumsToReturn) {
+                if (docsumFields == null) continue;
+
                 Cursor docsumItem = docsums.addObject();
                 Cursor docsum = docsumItem.setObject("docsum");
                 for (Map.Entry<String, Object> field : docsumFields.entrySet()) {