diff options
7 files changed, 70 insertions, 22 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 5e7c0d9a7a0..1ec72c6a2c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -213,6 +213,8 @@ public class VespaMetricSet { metrics.add(new Metric("hits_per_query.average")); metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); + metrics.add(new Metric("dispatch_internal.rate")); + metrics.add(new Metric("dispatch_fdispatch.rate")); metrics.add(new Metric("totalhits_per_query.average")); metrics.add(new Metric("empty_results.rate")); diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index db724b41657..e850312158c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -11,14 +11,14 @@ import com.yahoo.concurrent.Receiver.MessageState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; +import com.yahoo.jdisc.Metric; import com.yahoo.net.HostName; -import com.yahoo.search.dispatch.Dispatcher; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.FastSearcher; import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -26,6 +26,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; @@ -33,6 +34,7 @@ import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; +import org.apache.commons.lang.StringUtils; import java.net.InetAddress; import java.net.UnknownHostException; @@ -48,11 +50,8 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; -import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.lang.StringUtils; - import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; /** @@ -101,12 +100,13 @@ public class ClusterSearcher extends Searcher { DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, Statistics manager, + Metric metric, FS4ResourcePool fs4ResourcePool, VipStatus vipStatus) { super(id); this.fs4ResourcePool = fs4ResourcePool; - Dispatcher dispatcher = new Dispatcher(id.stringValue(), dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus); + Dispatcher dispatcher = new Dispatcher(id.stringValue(), dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus, metric); monitor = (dispatcher.searchCluster().directDispatchTarget().isPresent()) // dispatcher should decide vip status instead ? new ClusterMonitor(this, monitorConfig, Optional.empty()) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 0aee51e1e32..cf5bcedcf51 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -3,6 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.component.AbstractComponent; import com.yahoo.container.handler.VipStatus; +import com.yahoo.jdisc.Metric; import com.yahoo.prelude.fastsearch.FS4InvokerFactory; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -38,6 +39,9 @@ import java.util.Set; * @author ollvir */ public class Dispatcher extends AbstractComponent { + private static final String FDISPATCH_METRIC = "dispatch_fdispatch"; + private static final String INTERNAL_METRIC = "dispatch_internal"; + private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; /** If enabled, this internal dispatcher will be preferred over fdispatch whenever possible */ @@ -56,17 +60,23 @@ public class Dispatcher extends AbstractComponent { private final FS4InvokerFactory fs4InvokerFactory; private final RpcInvokerFactory rpcInvokerFactory; - public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + private final Metric metric; + private final Metric.Context metricContext; + + public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, + VipStatus vipStatus, Metric metric) { this(new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus), dispatchConfig, - fs4ResourcePool, new RpcResourcePool(dispatchConfig)); + fs4ResourcePool, new RpcResourcePool(dispatchConfig), metric); } - public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, RpcResourcePool rpcResourcePool) { + public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, + RpcResourcePool rpcResourcePool, Metric metric) { this(searchCluster, dispatchConfig, new FS4InvokerFactory(fs4ResourcePool, searchCluster), - new RpcInvokerFactory(rpcResourcePool, searchCluster)); + new RpcInvokerFactory(rpcResourcePool, searchCluster), metric); } - public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4InvokerFactory fs4InvokerFactory, RpcInvokerFactory rpcInvokerFactory) { + public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4InvokerFactory fs4InvokerFactory, + RpcInvokerFactory rpcInvokerFactory, Metric metric) { this.searchCluster = searchCluster; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); @@ -75,6 +85,9 @@ public class Dispatcher extends AbstractComponent { this.fs4InvokerFactory = fs4InvokerFactory; this.rpcInvokerFactory = rpcInvokerFactory; + + this.metric = metric; + this.metricContext = metric.createContext(null); } /** Returns the search cluster this dispatches to */ @@ -102,7 +115,8 @@ public class Dispatcher extends AbstractComponent { } public Optional<SearchInvoker> getSearchInvoker(Query query, VespaBackEndSearcher searcher) { - if (multilevelDispatch || ! query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) { + if (multilevelDispatch || !query.properties().getBoolean(dispatchInternal, internalDispatchByDefault)) { + emitDispatchMetric(Optional.empty()); return Optional.empty(); } @@ -117,6 +131,7 @@ public class Dispatcher extends AbstractComponent { query.setHits(0); query.setOffset(0); } + emitDispatchMetric(invoker); return invoker; } @@ -127,7 +142,7 @@ public class Dispatcher extends AbstractComponent { // build invoker based on searchpath private Optional<SearchInvoker> getSearchPathInvoker(Query query, InvokerFactory invokerFactory, VespaBackEndSearcher searcher) { String searchPath = query.getModel().getSearchPath(); - if(searchPath == null) { + if (searchPath == null) { return Optional.empty(); } try { @@ -181,4 +196,12 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } + + private void emitDispatchMetric(Optional<SearchInvoker> invoker) { + if (invoker.isEmpty()) { + metric.add(FDISPATCH_METRIC, 1, metricContext); + } else { + metric.add(INTERNAL_METRIC, 1, metricContext); + } + } } diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 886fd86c5aa..1e19a1397e0 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -5,6 +5,8 @@ import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.component.ComponentId; import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.handler.VipStatus; +import com.yahoo.container.protect.Error; import com.yahoo.container.search.Fs4Config; import com.yahoo.fs4.QueryPacket; import com.yahoo.prelude.IndexFacts; @@ -14,12 +16,11 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.test.MockMetric; import com.yahoo.search.Query; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; -import com.yahoo.container.handler.VipStatus; -import com.yahoo.container.protect.Error; import com.yahoo.statistics.Statistics; import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.Test; @@ -542,6 +543,7 @@ public class ClusterSearcherTestCase { new DispatchConfig.Builder().build(), createClusterInfoConfig(), Statistics.nullImplementation, + new MockMetric(), new FS4ResourcePool(new Fs4Config.Builder().build(), new QrConfig.Builder().build()), new VipStatus()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 001c8d57759..6be895f33d2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -23,7 +23,7 @@ class MockDispatcher extends Dispatcher { public MockDispatcher(String clusterId, List<Node> nodes, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { - super(clusterId, toDispatchConfig(nodes), fs4ResourcePool, containerClusterSize, vipStatus); + super(clusterId, toDispatchConfig(nodes), fs4ResourcePool, containerClusterSize, vipStatus, new MockMetric()); } private static DispatchConfig toDispatchConfig(List<Node> nodes) { @@ -43,5 +43,4 @@ class MockDispatcher extends Dispatcher { public void fill(Result result, String summaryClass) { } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockMetric.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockMetric.java new file mode 100644 index 00000000000..213a362edd5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockMetric.java @@ -0,0 +1,21 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + +import com.yahoo.jdisc.Metric; + +import java.util.Map; + +public class MockMetric implements Metric { + @Override + public void set(String key, Number val, Context ctx) { + } + + @Override + public void add(String key, Number val, Context ctx) { + } + + @Override + public Context createContext(Map<String, ?> properties) { + return null; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 0cc58801298..859e10dbe2c 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -3,6 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.FS4InvokerFactory; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.test.MockMetric; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; @@ -38,7 +39,7 @@ public class DispatcherTest { builder.useMultilevelDispatch(true); DispatchConfig dc = new DispatchConfig(builder); - Dispatcher disp = new Dispatcher(cl, dc, new MockFS4InvokerFactory(cl), new MockRpcInvokerFactory()); + Dispatcher disp = new Dispatcher(cl, dc, new MockFS4InvokerFactory(cl), new MockRpcInvokerFactory(), new MockMetric()); assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false)); } @@ -52,7 +53,7 @@ public class DispatcherTest { assertThat(nodes.get(0).key(), is(2)); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory()); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(q, null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -67,7 +68,7 @@ public class DispatcherTest { } }; MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> true); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory()); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -84,7 +85,7 @@ public class DispatcherTest { assertThat(acceptIncompleteCoverage, is(true)); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory()); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, new MockRpcInvokerFactory(), new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(true)); invokerFactory.verifyAllEventsProcessed(); @@ -95,7 +96,7 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 1); MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, null); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, null, new MockMetric()); Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); assertThat(invoker.isPresent(), is(false)); invokerFactory.verifyAllEventsProcessed(); |