diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
6 files changed, 78 insertions, 7 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index ee064123b1b..a5c6f3650e0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -4,6 +4,8 @@ package com.yahoo.search.dispatch; import com.yahoo.component.AbstractComponent; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; +import com.yahoo.prelude.fastsearch.FS4PingFactory; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -51,6 +53,9 @@ public class Dispatcher extends AbstractComponent { private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; + /** If enabled, this internal dispatcher will be preferred over fdispatch whenever possible */ + public static final CompoundName dispatchInternal = CompoundName.fromComponents(DISPATCH, INTERNAL); + /** If enabled, search queries will use protobuf rpc */ public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF); @@ -59,6 +64,7 @@ public class Dispatcher extends AbstractComponent { private final LoadBalancer loadBalancer; private final boolean multilevelDispatch; + private final boolean internalDispatchByDefault; private final InvokerFactory invokerFactory; @@ -80,13 +86,15 @@ public class Dispatcher extends AbstractComponent { public static Dispatcher create(String clusterId, DispatchConfig dispatchConfig, + FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus, Metric metric) { var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus); var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, !dispatchConfig.useFdispatchByDefault()); + var pingFactory = dispatchConfig.useFdispatchByDefault()? new FS4PingFactory(fs4ResourcePool) : rpcFactory; - return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, rpcFactory, metric); + return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, pingFactory, metric); } public Dispatcher(SearchCluster searchCluster, @@ -97,8 +105,9 @@ public class Dispatcher extends AbstractComponent { this.searchCluster = searchCluster; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); - this.invokerFactory = invokerFactory; this.multilevelDispatch = dispatchConfig.useMultilevelDispatch(); + this.internalDispatchByDefault = !dispatchConfig.useFdispatchByDefault(); + this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); @@ -124,7 +133,7 @@ public class Dispatcher extends AbstractComponent { } public Optional<SearchInvoker> getSearchInvoker(Query query, VespaBackEndSearcher searcher) { - if (multilevelDispatch) { + if (multilevelDispatch || ! query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) { emitDispatchMetric(Optional.empty()); return Optional.empty(); } @@ -139,6 +148,7 @@ public class Dispatcher extends AbstractComponent { query.setOffset(0); } emitDispatchMetric(invoker); + query.properties().set(dispatchInternal, true); return invoker; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index ce4891d7c25..cd8228624c5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -39,6 +39,9 @@ public class InvokerResult { if (hit.hasSortData()) { fh.setSortData(hit.getSortData(), sorting); } + if (hit.getQueryPacketData() != null) { + fh.setQueryPacketData(hit.getQueryPacketData()); + } fh.setQuery(query); fh.setFillable(); fh.setCached(false); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 0749d8476c2..63302bee8c1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -1,5 +1,6 @@ package com.yahoo.search.dispatch; +import com.yahoo.fs4.QueryPacketData; import java.util.Arrays; @@ -9,7 +10,8 @@ public class LeanHit implements Comparable<LeanHit> { private final byte [] sortData; private final int partId; private final int distributionKey; - + //TODO Remove when FS4 is gone + private QueryPacketData queryPacketData; public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { this.gid = gid; this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; @@ -31,6 +33,12 @@ public class LeanHit implements Comparable<LeanHit> { public int getPartId() { return partId; } public int getDistributionKey() { return distributionKey; } + QueryPacketData getQueryPacketData() { return queryPacketData; } + + public void setQueryPacketData(QueryPacketData queryPacketData) { + this.queryPacketData = queryPacketData; + } + @Override public int compareTo(LeanHit o) { int res = (sortData != null) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java new file mode 100644 index 00000000000..dea6f741bb0 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -0,0 +1,40 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; +import com.yahoo.prelude.fastsearch.FastSearcher; +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.yolean.Exceptions; + +import java.util.concurrent.Callable; + +/** + * @author bratseth + * @author ollivir + */ +public class Pinger implements Callable<Pong> { + private final Node node; + private final ClusterMonitor<Node> clusterMonitor; + private final FS4ResourcePool fs4ResourcePool; + + public Pinger(Node node, ClusterMonitor<Node> clusterMonitor, FS4ResourcePool fs4ResourcePool) { + this.node = node; + this.clusterMonitor = clusterMonitor; + this.fs4ResourcePool = fs4ResourcePool; + } + + public Pong call() { + try { + Pong pong = FastSearcher.ping(new Ping(clusterMonitor.getConfiguration().getRequestTimeout()), + fs4ResourcePool.getBackend(node.hostname(), node.fs4port()), node.toString()); + return pong; + } catch (RuntimeException e) { + return new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + + Exceptions.toMessageString(e))); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index dc35ebdc65f..3657e0b5c76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -186,11 +186,17 @@ public class SearchCluster implements NodeManager<Node> { } /** + * Returns the nodes of this cluster as an immutable map indexed by host. + * One host may contain multiple nodes (on different ports), so this is a multi-map. + */ + public ImmutableMultimap<String, Node> nodesByHost() { return nodesByHost; } + + /** * Returns the recipient we should dispatch queries directly to (bypassing fdispatch), * or empty if we should not dispatch directly. */ public Optional<Node> directDispatchTarget() { - if ( directDispatchTarget.isEmpty()) return Optional.empty(); + if ( ! directDispatchTarget.isPresent()) return Optional.empty(); // Only use direct dispatch if the local group has sufficient coverage Group localSearchGroup = groups.get(directDispatchTarget.get().group()); @@ -234,11 +240,13 @@ public class SearchCluster implements NodeManager<Node> { } private boolean usesDirectDispatchTo(Node node) { - return directDispatchTarget.isPresent() && directDispatchTarget.get().equals(node); + if ( ! directDispatchTarget.isPresent()) return false; + return directDispatchTarget.get().equals(node); } private boolean usesDirectDispatchTo(Group group) { - return directDispatchTarget.isPresent() && directDispatchTarget.get().group() == group.id(); + if ( ! directDispatchTarget.isPresent()) return false; + return directDispatchTarget.get().group() == group.id(); } /** Used by the cluster monitor to manage node status */ diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java index e8f4d566028..cfed5ed00ad 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java @@ -54,6 +54,8 @@ class HitConverter implements ResultBuilder.HitConverter { throw new NullPointerException("Hit has no context."); hit.setSource(hitContext.getSource()); hit.setQuery(hitContext.getQuery()); + if (hitContext.getQueryPacketData() != null) + hit.setQueryPacketData(hitContext.getQueryPacketData()); return hit; } |