Revert "Balder/no more fs4 dispatching from fastsearcher"

6 files changed, 78 insertions, 7 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index ee064123b1b..a5c6f3650e0 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -4,6 +4,8 @@ package com.yahoo.search.dispatch;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.prelude.fastsearch.FS4PingFactory;
+import com.yahoo.prelude.fastsearch.FS4ResourcePool;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
 import com.yahoo.processing.request.CompoundName;
 import com.yahoo.search.Query;
@@ -51,6 +53,9 @@ public class Dispatcher extends AbstractComponent {
 
     private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3;
 
+    /** If enabled, this internal dispatcher will be preferred over fdispatch whenever possible */
+    public static final CompoundName dispatchInternal = CompoundName.fromComponents(DISPATCH, INTERNAL);
+
     /** If enabled, search queries will use protobuf rpc */
     public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF);
 
@@ -59,6 +64,7 @@ public class Dispatcher extends AbstractComponent {
 
     private final LoadBalancer loadBalancer;
     private final boolean multilevelDispatch;
+    private final boolean internalDispatchByDefault;
 
     private final InvokerFactory invokerFactory;
 
@@ -80,13 +86,15 @@ public class Dispatcher extends AbstractComponent {
 
     public static Dispatcher create(String clusterId,
                                     DispatchConfig dispatchConfig,
+                                    FS4ResourcePool fs4ResourcePool,
                                     int containerClusterSize,
                                     VipStatus vipStatus,
                                     Metric metric) {
         var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus);
         var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, !dispatchConfig.useFdispatchByDefault());
+        var pingFactory = dispatchConfig.useFdispatchByDefault()? new FS4PingFactory(fs4ResourcePool) : rpcFactory;
 
-        return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, rpcFactory, metric);
+        return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, pingFactory, metric);
     }
 
     public Dispatcher(SearchCluster searchCluster,
@@ -97,8 +105,9 @@ public class Dispatcher extends AbstractComponent {
         this.searchCluster = searchCluster;
         this.loadBalancer = new LoadBalancer(searchCluster,
                                   dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
-        this.invokerFactory = invokerFactory;
         this.multilevelDispatch = dispatchConfig.useMultilevelDispatch();
+        this.internalDispatchByDefault = !dispatchConfig.useFdispatchByDefault();
+        this.invokerFactory = invokerFactory;
         this.metric = metric;
         this.metricContext = metric.createContext(null);
 
@@ -124,7 +133,7 @@ public class Dispatcher extends AbstractComponent {
     }
 
     public Optional<SearchInvoker> getSearchInvoker(Query query, VespaBackEndSearcher searcher) {
-        if (multilevelDispatch) {
+        if (multilevelDispatch || ! query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) {
             emitDispatchMetric(Optional.empty());
             return Optional.empty();
         }
@@ -139,6 +148,7 @@ public class Dispatcher extends AbstractComponent {
             query.setOffset(0);
         }
         emitDispatchMetric(invoker);
+        query.properties().set(dispatchInternal, true);
 
         return invoker;
     }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java
index ce4891d7c25..cd8228624c5 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java
@@ -39,6 +39,9 @@ public class InvokerResult {
             if (hit.hasSortData()) {
                 fh.setSortData(hit.getSortData(), sorting);
             }
+            if (hit.getQueryPacketData() != null) {
+                fh.setQueryPacketData(hit.getQueryPacketData());
+            }
             fh.setQuery(query);
             fh.setFillable();
             fh.setCached(false);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
index 0749d8476c2..63302bee8c1 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
@@ -1,5 +1,6 @@
 package com.yahoo.search.dispatch;
 
+import com.yahoo.fs4.QueryPacketData;
 
 import java.util.Arrays;
 
@@ -9,7 +10,8 @@ public class LeanHit implements Comparable<LeanHit> {
     private final byte [] sortData;
     private final int partId;
     private final int distributionKey;
-
+    //TODO Remove when FS4 is gone
+    private QueryPacketData queryPacketData;
     public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) {
         this.gid = gid;
         this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance;
@@ -31,6 +33,12 @@ public class LeanHit implements Comparable<LeanHit> {
     public int getPartId() { return partId; }
     public int getDistributionKey() { return distributionKey; }
 
+    QueryPacketData getQueryPacketData() { return queryPacketData; }
+
+    public void setQueryPacketData(QueryPacketData queryPacketData) {
+        this.queryPacketData = queryPacketData;
+    }
+
     @Override
     public int compareTo(LeanHit o) {
         int res = (sortData != null)
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
new file mode 100644
index 00000000000..dea6f741bb0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
@@ -0,0 +1,40 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch.searchcluster;
+
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.prelude.fastsearch.FS4ResourcePool;
+import com.yahoo.prelude.fastsearch.FastSearcher;
+import com.yahoo.search.cluster.ClusterMonitor;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.yolean.Exceptions;
+
+import java.util.concurrent.Callable;
+
+/**
+ * @author bratseth
+ * @author ollivir
+ */
+public class Pinger implements Callable<Pong> {
+    private final Node node;
+    private final ClusterMonitor<Node> clusterMonitor;
+    private final FS4ResourcePool fs4ResourcePool;
+
+    public Pinger(Node node, ClusterMonitor<Node> clusterMonitor, FS4ResourcePool fs4ResourcePool) {
+        this.node = node;
+        this.clusterMonitor = clusterMonitor;
+        this.fs4ResourcePool = fs4ResourcePool;
+    }
+
+    public Pong call() {
+        try {
+            Pong pong = FastSearcher.ping(new Ping(clusterMonitor.getConfiguration().getRequestTimeout()),
+                                          fs4ResourcePool.getBackend(node.hostname(), node.fs4port()), node.toString());
+            return pong;
+        } catch (RuntimeException e) {
+            return new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": "
+                            + Exceptions.toMessageString(e)));
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index dc35ebdc65f..3657e0b5c76 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -186,11 +186,17 @@ public class SearchCluster implements NodeManager<Node> {
     }
 
     /**
+     * Returns the nodes of this cluster as an immutable map indexed by host.
+     * One host may contain multiple nodes (on different ports), so this is a multi-map.
+     */
+    public ImmutableMultimap<String, Node> nodesByHost() { return nodesByHost; }
+
+    /**
      * Returns the recipient we should dispatch queries directly to (bypassing fdispatch),
      * or empty if we should not dispatch directly.
      */
     public Optional<Node> directDispatchTarget() {
-        if ( directDispatchTarget.isEmpty()) return Optional.empty();
+        if ( ! directDispatchTarget.isPresent()) return Optional.empty();
 
         // Only use direct dispatch if the local group has sufficient coverage
         Group localSearchGroup = groups.get(directDispatchTarget.get().group());
@@ -234,11 +240,13 @@ public class SearchCluster implements NodeManager<Node> {
     }
 
     private boolean usesDirectDispatchTo(Node node) {
-        return directDispatchTarget.isPresent() && directDispatchTarget.get().equals(node);
+        if ( ! directDispatchTarget.isPresent()) return false;
+        return directDispatchTarget.get().equals(node);
     }
 
     private boolean usesDirectDispatchTo(Group group) {
-        return directDispatchTarget.isPresent() && directDispatchTarget.get().group() == group.id();
+        if ( ! directDispatchTarget.isPresent()) return false;
+        return directDispatchTarget.get().group() == group.id();
     }
 
     /** Used by the cluster monitor to manage node status */
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java
index e8f4d566028..cfed5ed00ad 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java
@@ -54,6 +54,8 @@ class HitConverter implements ResultBuilder.HitConverter {
             throw new NullPointerException("Hit has no context.");
         hit.setSource(hitContext.getSource());
         hit.setQuery(hitContext.getQuery());
+        if (hitContext.getQueryPacketData() != null)
+            hit.setQueryPacketData(hitContext.getQueryPacketData());
         return hit;
     }