Reimplement partial coverage reporting

5 files changed, 68 insertions, 17 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java
index 51048db3cb7..f5d082635ab 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java
@@ -8,9 +8,12 @@ import com.yahoo.search.Result;
 import com.yahoo.search.dispatch.FillInvoker;
 import com.yahoo.search.dispatch.InterleavedFillInvoker;
 import com.yahoo.search.dispatch.InterleavedSearchInvoker;
+import com.yahoo.search.dispatch.SearchErrorInvoker;
 import com.yahoo.search.dispatch.SearchInvoker;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
+import com.yahoo.search.result.Coverage;
+import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.result.Hit;
 
 import java.util.ArrayList;
@@ -70,19 +73,24 @@ public class FS4InvokerFactory {
         Map<Integer, SearchInvoker> invokers = new HashMap<>();
         Set<Integer> failed = null;
         for (Node node : nodes) {
+            boolean nodeAdded = false;
             if (node.isWorking()) {
                 Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port());
                 if (backend.probeConnection()) {
                     invokers.put(node.key(), new FS4SearchInvoker(searcher, query, backend.openChannel(), node));
-                } else {
-                    if(failed == null) {
-                        failed = new HashSet<>();
-                    }
-                    failed.add(node.key());
+                    nodeAdded = true;
+                }
+            }
+
+            if (!nodeAdded) {
+                if (failed == null) {
+                    failed = new HashSet<>();
                 }
+                failed.add(node.key());
             }
         }
-        if (failed != null && ! acceptIncompleteCoverage) {
+
+        if (failed != null) {
             List<Node> success = new ArrayList<>(nodes.size() - failed.size());
             for (Node node : nodes) {
                 if (!failed.contains(node.key())) {
@@ -90,9 +98,14 @@ public class FS4InvokerFactory {
                 }
             }
             if (!searchCluster.isPartialGroupCoverageSufficient(groupId, success)) {
-                return Optional.empty();
+                if (acceptIncompleteCoverage) {
+                    createCoverageErrorInvoker(invokers, nodes, failed);
+                } else {
+                    return Optional.empty();
+                }
             }
         }
+
         if (invokers.size() == 1) {
             return Optional.of(invokers.values().iterator().next());
         } else {
@@ -100,6 +113,25 @@ public class FS4InvokerFactory {
         }
     }
 
+    private void createCoverageErrorInvoker(Map<Integer, SearchInvoker> invokers, List<Node> nodes, Set<Integer> failed) {
+        long activeDocuments = 0;
+        StringBuilder down = new StringBuilder("Connection failure on nodes with distribution-keys: ");
+        Integer key = null;
+        for (Node node : nodes) {
+            if (failed.contains(node.key())) {
+                activeDocuments += node.getActiveDocuments();
+                if (key == null) {
+                    key = node.key();
+                } else {
+                    down.append(", ");
+                }
+                down.append(node.key());
+            }
+        }
+        Coverage coverage = new Coverage(0, activeDocuments, 0);
+        invokers.put(key, new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError(down.toString()), coverage));
+    }
+
     public FillInvoker getFillInvoker(Query query, Node node) {
         return new FS4FillInvoker(searcher, query, fs4ResourcePool, node.hostname(), node.fs4port(), node.key());
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
index 11b71c2c159..a98c34295ee 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
@@ -272,14 +272,20 @@ public class FastSearcher extends VespaBackEndSearcher {
         Result result = new Result(query);
         // keep a separate tally of coverage as the normal merge counts using
         // federated query rules
-        Coverage finalCoverage = new Coverage(0, 0);
+        Coverage finalCoverage = null;
 
         for (Result partialResult : results) {
-            finalCoverage.mergeWithPartition(partialResult.getCoverage(true));
+            if(finalCoverage == null) {
+                finalCoverage = partialResult.getCoverage(true);
+            } else {
+                finalCoverage.mergeWithPartition(partialResult.getCoverage(true));
+            }
             result.mergeWith(partialResult);
             result.hits().addAll(partialResult.hits().asUnorderedHits());
         }
-        result.setCoverage(finalCoverage);
+        if (finalCoverage != null) {
+            result.setCoverage(finalCoverage);
+        }
 
         if (query.getOffset() != 0 || result.hits().size() > query.getHits()) {
             // with multiple results, each partial result is expected to have
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index f83d5eb23f6..708d80d4969 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -14,6 +14,7 @@ import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException;
 import com.yahoo.search.dispatch.searchcluster.Group;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
+import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.vespa.config.search.DispatchConfig;
 
 import java.util.Arrays;
@@ -125,7 +126,7 @@ public class Dispatcher extends AbstractComponent {
                 return invokerFactory.supply(query, -1, nodes, true);
             }
         } catch (InvalidSearchPathException e) {
-            return Optional.of(new SearchErrorInvoker(e.getMessage()));
+            return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage())));
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java
index b716c182615..d5c505aa31b 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java
@@ -5,6 +5,7 @@ import com.yahoo.fs4.QueryPacket;
 import com.yahoo.prelude.fastsearch.CacheKey;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
+import com.yahoo.search.result.Coverage;
 import com.yahoo.search.result.ErrorMessage;
 
 import java.io.IOException;
@@ -12,17 +13,24 @@ import java.util.Arrays;
 import java.util.List;
 
 /**
- * A search invoker that will immediately produce an error that occurred during invoker construction.
- * Currently used for invalid searchpath values.
+ * A search invoker that will immediately produce an error that occurred during
+ * invoker construction. Currently used for invalid searchpath values and node
+ * failure
  *
  * @author ollivir
  */
 public class SearchErrorInvoker extends SearchInvoker {
-    private final String message;
+    private final ErrorMessage message;
     private Query query;
+    private final Coverage coverage;
 
-    public SearchErrorInvoker(String message) {
+    public SearchErrorInvoker(ErrorMessage message, Coverage coverage) {
         this.message = message;
+        this.coverage = coverage;
+    }
+
+    public SearchErrorInvoker(ErrorMessage message) {
+        this(message, null);
     }
 
     @Override
@@ -32,7 +40,11 @@ public class SearchErrorInvoker extends SearchInvoker {
 
     @Override
     protected List<Result> getSearchResults(CacheKey cacheKey) throws IOException {
-        return Arrays.asList(new Result(query, ErrorMessage.createIllegalQuery(message)));
+        Result res = new Result(query, message);
+        if (coverage != null) {
+            res.setCoverage(coverage);
+        }
+        return Arrays.asList(res);
     }
 
     @Override
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index b0e63d20931..b51620a978b 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -323,7 +323,7 @@ public class SearchCluster implements NodeManager<Node> {
      */
     public boolean isPartialGroupCoverageSufficient(int groupId, List<Node> nodes) {
         if (orderedGroups.size() == 1) {
-            return true;
+            return nodes.size() >= groupSize() - maxNodesDownPerGroup;
         }
         long sumOfActiveDocuments = 0;
         int otherGroups = 0;