From 4044f6b8a183c7792d043090c08c3106ed74a656 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 12 Apr 2021 16:04:19 +0200 Subject: Non-functional changes only --- .../yahoo/search/dispatch/CloseableInvoker.java | 2 ++ .../java/com/yahoo/search/dispatch/Dispatcher.java | 32 ++++++++-------------- .../com/yahoo/search/dispatch/FillInvoker.java | 4 ++- .../search/dispatch/InterleavedSearchInvoker.java | 12 +------- .../com/yahoo/search/dispatch/InvokerFactory.java | 2 +- .../com/yahoo/search/dispatch/InvokerResult.java | 7 +++++ .../java/com/yahoo/search/dispatch/LeanHit.java | 6 ++++ .../com/yahoo/search/dispatch/LoadBalancer.java | 6 ++-- .../com/yahoo/search/dispatch/ResponseMonitor.java | 2 ++ .../yahoo/search/dispatch/SearchErrorInvoker.java | 1 + .../java/com/yahoo/search/dispatch/SearchPath.java | 12 +++----- .../com/yahoo/search/dispatch/TopKEstimator.java | 12 ++++++++ .../search/dispatch/rpc/ProtobufSerialization.java | 4 +-- .../yahoo/search/dispatch/rpc/RpcPingFactory.java | 4 +++ .../dispatch/rpc/RpcProtobufFillInvoker.java | 1 + .../yahoo/search/dispatch/rpc/RpcResourcePool.java | 1 + .../search/dispatch/rpc/RpcSearchInvoker.java | 4 +-- .../search/dispatch/searchcluster/PingFactory.java | 4 ++- .../search/dispatch/searchcluster/Pinger.java | 2 ++ .../search/dispatch/searchcluster/PongHandler.java | 2 ++ 20 files changed, 70 insertions(+), 50 deletions(-) (limited to 'container-search') diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java index 515d6249fd8..9329f4a6819 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java @@ -12,6 +12,7 @@ import java.util.function.BiConsumer; * @author ollivir */ public abstract class CloseableInvoker implements Closeable { + protected abstract void release(); private BiConsumer teardown = null; @@ -35,4 +36,5 @@ public abstract class CloseableInvoker implements Closeable { } release(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 626cf087aca..9b92a78a7c9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -65,7 +65,7 @@ public class Dispatcher extends AbstractComponent { /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; - private final ClusterMonitor clusterMonitor; + private final ClusterMonitor clusterMonitor; private final LoadBalancer loadBalancer; @@ -108,7 +108,7 @@ public class Dispatcher extends AbstractComponent { } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(ClusterMonitor clusterMonitor, + protected Dispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, @@ -125,12 +125,7 @@ public class Dispatcher extends AbstractComponent { this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); searchCluster.addMonitoring(clusterMonitor); - Thread warmup = new Thread(new Runnable() { - @Override - public void run() { - warmup(dispatchConfig.warmuptime()); - } - }); + Thread warmup = new Thread(() -> warmup(dispatchConfig.warmuptime())); warmup.start(); try { while ( ! searchCluster.hasInformationAboutAllNodes()) { @@ -139,20 +134,17 @@ public class Dispatcher extends AbstractComponent { warmup.join(); } catch (InterruptedException e) {} - /* - * No we have information from all nodes and a ping iteration has completed. - * Instead of waiting until next ping interval to update coverage and group state, - * we should compute the state ourselves, so that when the dispatcher is ready the state - * of its groups are also known. - */ + // Now we have information from all nodes and a ping iteration has completed. + // Instead of waiting until next ping interval to update coverage and group state, + // we should compute the state ourselves, so that when the dispatcher is ready the state + // of its groups are also known. searchCluster.pingIterationCompleted(); } - /* - Will run important code in order to trigger JIT compilation and avoid cold start issues. - Currently warms up lz4 compression code. + /** + * Will run important code in order to trigger JIT compilation and avoid cold start issues. + * Currently warms up lz4 compression code. */ - private static long warmup(double seconds) { return new Compressor().warmup(seconds); } @@ -164,7 +156,7 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + // The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. clusterMonitor.shutdown(); invokerFactory.release(); } @@ -212,7 +204,7 @@ public class Dispatcher extends AbstractComponent { return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), - Arrays.asList(node), + List.of(node), true, maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java index dd4c4494ac5..8b7714aaf3b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java @@ -10,7 +10,8 @@ import com.yahoo.search.Result; * @author ollivir */ public abstract class FillInvoker extends CloseableInvoker { - /** Retrieve document summaries for the unfilled hits in the given {@link Result} */ + + /** Retrieves document summaries for the unfilled hits in the given {@link Result} */ public void fill(Result result, String summaryClass) { sendFillRequest(result, summaryClass); getFillResults(result, summaryClass); @@ -19,4 +20,5 @@ public abstract class FillInvoker extends CloseableInvoker { protected abstract void getFillResults(Result result, String summaryClass); protected abstract void sendFillRequest(Result result, String summaryClass); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 036592dcf23..adf7368faa2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -235,17 +235,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private String dbg(LeanHit hit) { - var buf = new StringBuilder(); - buf.append("LeanHit["); - if (hit.hasSortData()) buf.append("hasSortData,"); - buf.append("relevance=").append(hit.getRelevance()); - buf.append(",partId=").append(hit.getPartId()); - buf.append(",distributionKey=").append(hit.getDistributionKey()); - buf.append("]"); - return buf.toString(); - } - private List mergeResult(Result result, InvokerResult partialResult, List current) { collectCoverage(partialResult.getResult().getCoverage(true)); @@ -382,4 +371,5 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM // For testing Collection invokers() { return invokers; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f65e0e43757..dcf052d28e6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -35,7 +35,7 @@ public abstract class InvokerFactory { public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); /** - * Create a {@link SearchInvoker} for a list of content nodes. + * Creates a {@link SearchInvoker} for a list of content nodes. * * @param searcher the searcher processing the query * @param query the search query being processed diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index 94c347a6927..2723429c0cf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -12,14 +12,19 @@ import java.util.List; /** * Wraps a Result and a flat, skinny hit list + * + * @author baldersheim */ public class InvokerResult { + private final Result result; private final List leanHits; + public InvokerResult(Result result) { this.result = result; this.leanHits = Collections.emptyList(); } + public InvokerResult(Query query, int expectedHits) { result = new Result(query); leanHits = new ArrayList<>(expectedHits); @@ -32,6 +37,7 @@ public class InvokerResult { public List getLeanHits() { return leanHits; } + void complete() { Query query = result.getQuery(); Sorting sorting = query.getRanking().getSorting(); @@ -47,4 +53,5 @@ public class InvokerResult { } leanHits.clear(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 8a90557fa3b..df8fb2f29fa 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -4,7 +4,11 @@ package com.yahoo.search.dispatch; import java.util.Arrays; +/** + * @author baldersheim + */ public class LeanHit implements Comparable { + private final byte [] gid; private final double relevance; private final byte [] sortData; @@ -21,6 +25,7 @@ public class LeanHit implements Comparable { this.partId = partId; this.distributionKey = distributionKey; } + public double getRelevance() { return relevance; } public byte [] getGid() { return gid; } public byte [] getSortData() { return sortData; } @@ -49,4 +54,5 @@ public class LeanHit implements Comparable { int vr = (int) right[i] & 0xFF; return vl - vr; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 05e1ea6e2f9..ebde2ffc611 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -43,8 +43,8 @@ public class LoadBalancer { } /** - * Select and allocate the search cluster group which is to be used for the next search query. Callers must call - * {@link #releaseGroup} symmetrically for each taken allocation. + * Select and allocate the search cluster group which is to be used for the next search query. + * Callers must call {@link #releaseGroup} symmetrically for each taken allocation. * * @param rejectedGroups if not null, the load balancer will only return groups with IDs not in the set * @return the node group to target, or empty if the internal dispatch logic cannot be used @@ -76,7 +76,7 @@ public class LoadBalancer { synchronized (this) { for (GroupStatus sched : scoreboard) { if (sched.group.id() == group.id()) { - sched.release(success, (double) searchTimeMs / 1000.0); + sched.release(success, searchTimeMs / 1000.0); break; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java index c2e81d43677..3ebd21fa18a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java @@ -9,5 +9,7 @@ package com.yahoo.search.dispatch; * @author ollivir */ public interface ResponseMonitor { + void responseAvailable(T from); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 256759360f7..7dbc2e98759 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -61,4 +61,5 @@ public class SearchErrorInvoker extends SearchInvoker { protected void setMonitor(ResponseMonitor monitor) { this.monitor = monitor; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 7937be50813..f6480f80c01 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -28,15 +28,11 @@ import java.util.stream.IntStream; public class SearchPath { /** - * Parse the search path and select nodes from the given cluster based on it. + * Parses the search path and select nodes from the given cluster based on it. * - * @param searchPath - * unparsed search path expression (see: model.searchPath in Search - * API reference) - * @param cluster - * the search cluster from which nodes are selected - * @throws InvalidSearchPathException - * if the searchPath is malformed + * @param searchPath unparsed search path expression (see: model.searchPath in Search API reference) + * @param cluster the search cluster from which nodes are selected + * @throws InvalidSearchPathException if the searchPath is malformed * @return list of nodes chosen with the search path, or an empty list in which * case some other node selection logic should be used */ diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java index aef1ef2f498..315dfdd4320 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -6,9 +6,11 @@ import org.apache.commons.math3.distribution.TDistribution; /** * Use StudentT distribution and estimate how many hits you need from each partition * to to get the globally top-k documents with the desired probability + * * @author baldersheim */ public class TopKEstimator { + private final TDistribution studentT; private final double defaultP; private final boolean estimate; @@ -19,9 +21,11 @@ public class TopKEstimator { private static boolean needEstimate(double p) { return (0.0 < p) && (p < 1.0); } + TopKEstimator(double freedom, double defaultProbability) { this(freedom, defaultProbability, 0.0); } + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { this.studentT = new TDistribution(null, freedom); defaultP = defaultProbability; @@ -32,36 +36,44 @@ public class TopKEstimator { defaultCumulativeProbability[i] = computeCumulativeProbability(i+MIN_N, defaultP); } } + private double inverseCumulativeProbability(int n, double p) { if (p == defaultP && (n >= MIN_N) && (n < defaultCumulativeProbability.length + MIN_N)) { return defaultCumulativeProbability[n - MIN_N]; } return computeCumulativeProbability(n, p); } + private double computeCumulativeProbability(int n, double p) { double p_inverse = 1 - (1 - p)/computeN(n); return studentT.inverseCumulativeProbability(p_inverse); } + private double computeN(double n) { double p_max = (1 + skewFactor)/n; return Math.max(1, 1/p_max); } + double estimateExactK(double k, int n_i, double p) { double n = computeN(n_i); double variance = k * 1/n * (1 - 1/n); return k/n + inverseCumulativeProbability(n_i, p) * Math.sqrt(variance); } + double estimateExactK(double k, int n) { return estimateExactK(k, n, defaultP); } + public int estimateK(int k, int n) { return (estimate && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) : k; } + public int estimateK(int k, int n, double p) { return (needEstimate(p) && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) : k; } } + diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 6dc01f34571..250524fadf2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -196,7 +196,7 @@ public class ProtobufSerialization { result.getResult().setTotalHitCount(protobuf.getTotalHitCount()); result.getResult().setCoverage(convertToCoverage(protobuf)); - var haveGrouping = protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty(); + var haveGrouping = ! protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); int cnt = buf.getInt(null); @@ -219,7 +219,7 @@ public class ProtobufSerialization { } var slimeTrace = protobuf.getSlimeTrace(); - if (slimeTrace != null && !slimeTrace.isEmpty()) { + if ( ! slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); traces.add(new SlimeAdapter(BinaryFormat.decode(slimeTrace.toByteArray()).get())); query.trace(traces, query.getTraceLevel()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java index 7d9b3ca1034..01e3ec3ca2b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -8,12 +8,16 @@ import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.PongHandler; public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { this.rpcResourcePool = rpcResourcePool; } + @Override public Pinger createPinger(Node node, ClusterMonitor monitor, PongHandler pongHandler) { return new RpcPing(node, monitor, rpcResourcePool, pongHandler); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 341b9b2bce3..8a17be8102e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -38,6 +38,7 @@ import java.util.logging.Logger; * @author ollivir */ public class RpcProtobufFillInvoker extends FillInvoker { + private static final String RPC_METHOD = "vespa.searchprotocol.getDocsums"; private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index 746461630dd..c3d072b8db6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -26,6 +26,7 @@ import java.util.Random; * @author ollivir */ public class RpcResourcePool extends AbstractComponent { + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 4c0b77207d5..20b11efb470 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -102,9 +102,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe ProtobufResponse protobufResponse = response.response().get(); CompressionType compression = CompressionType.valueOf(protobufResponse.compression()); byte[] payload = resourcePool.compressor().decompress(protobufResponse.compressedPayload(), compression, protobufResponse.uncompressedSize()); - var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); - - return result; + return ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 2e07d8d61e6..3b9e9573367 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -3,7 +3,9 @@ package com.yahoo.search.dispatch.searchcluster; import com.yahoo.search.cluster.ClusterMonitor; - +/** + * @author ollivir + */ public interface PingFactory { Pinger createPinger(Node node, ClusterMonitor monitor, PongHandler pongHandler); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java index b4a7ccbf98c..681a7d0af2c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -8,5 +8,7 @@ package com.yahoo.search.dispatch.searchcluster; * @author baldersheim */ public interface Pinger { + void ping(); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java index 1b39f14fd86..c39426e9d76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -9,5 +9,7 @@ import com.yahoo.prelude.Pong; * @author baldersheim */ public interface PongHandler { + void handle(Pong pong); + } -- cgit v1.2.3