diff options
Diffstat (limited to 'container-search')
13 files changed, 543 insertions, 322 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 44c490b8ed1..4fc3f7919cf 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -11,6 +11,11 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.data.access.ArrayTraverser; +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.simple.Value; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.search.Searcher; @@ -104,14 +109,51 @@ public class JuniperSearcher extends Searcher { for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { if (index.getDynamicSummary() || index.getHighlightSummary()) { - HitField fieldValue = fastHit.buildHitField(index.getName(), true); - if (fieldValue != null) - insertTags(fieldValue, bolding, index.getDynamicSummary()); + var field = fastHit.getField(index.getName()); + if (StringArrayConverter.shouldHandleField(field)) { + new StringArrayConverter(fastHit, index, field, bolding); + } else { + HitField fieldValue = fastHit.buildHitField(index.getName(), true); + if (fieldValue != null) { + insertTags(fieldValue, bolding, index.getDynamicSummary()); + } + } } } } } + private class StringArrayConverter implements ArrayTraverser { + + private Index index; + private boolean bolding; + private Value.ArrayValue convertedField = new Value.ArrayValue(); + + /** + * This converts the backend binary highlighting of each item in an array of string field, + * and creates a new field that replaces the original. + */ + StringArrayConverter(FastHit hit, Index index, Object field, boolean bolding) { + this.index = index; + this.bolding = bolding; + ((Inspectable)field).inspect().traverse(this); + hit.setField(index.getName(), convertedField); + } + + static boolean shouldHandleField(Object field) { + return (field instanceof Inspectable) && + (((Inspectable)field).inspect().type() == Type.ARRAY); + } + + @Override + public void entry(int idx, Inspector inspector) { + // This is how HitField is instantiated in Hit.buildHitField() when forceNoPreTokenize=true. + var hitField = new HitField(index.getName(), inspector.asString(), false); + insertTags(hitField, bolding, index.getDynamicSummary()); + convertedField.add(hitField.getContent()); + } + } + private void insertTags(HitField field, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext();) { diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 6fae5c97cd2..6401945799b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -68,6 +68,7 @@ public class StatisticsSearcher extends Searcher { private static final String PEAK_QPS_METRIC = "peak_qps"; private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; + private static final String DOCS_TARGET_TOTAL_METRIC = "documents_target_total"; private static final String DEGRADED_QUERIES_METRIC = "degraded_queries"; private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; @@ -252,6 +253,7 @@ public class StatisticsSearcher extends Searcher { } metric.add(DOCS_COVERED_METRIC, queryCoverage.getDocs(), metricContext); metric.add(DOCS_TOTAL_METRIC, queryCoverage.getActive(), metricContext); + metric.add(DOCS_TARGET_TOTAL_METRIC, queryCoverage.getTargetActive(), metricContext); } int hitCount = result.getConcreteHitCount(); metric.set(HITS_PER_QUERY_METRIC, (double) hitCount, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java new file mode 100644 index 00000000000..fbc179a10fa --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java @@ -0,0 +1,69 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.concurrent.Timer; +import com.yahoo.search.Query; +import com.yahoo.vespa.config.search.DispatchConfig; + +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_ADAPTIVE_TIMEOUT; + +/** + * Computes next timeout based on how many responses has been received so far + * + * @author baldersheim + */ +class AdaptiveTimeoutHandler implements TimeoutHandler { + private final double minimumCoverage; + private final int askedNodes; + private final int minimumResponses; + private final Query query; + private final Timer timer; + private final DispatchConfig config; + + private long deadline; + private long adaptiveTimeoutMin; + private long adaptiveTimeoutMax; + boolean adaptiveTimeoutCalculated = false; + + AdaptiveTimeoutHandler(Timer timer, DispatchConfig config, int askedNodes, Query query) { + minimumCoverage = config.minSearchCoverage(); + this.config = config; + this.askedNodes = askedNodes; + this.query = query; + this.timer = timer; + minimumResponses = (int) Math.ceil(askedNodes * minimumCoverage / 100.0); + deadline = timer.milliTime() + query.getTimeLeft(); + } + + @Override + public long nextTimeoutMS(int answeredNodes) { + if (askedNodes == answeredNodes) return query.getTimeLeft(); // All nodes have responded - done + if (answeredNodes < minimumResponses) return query.getTimeLeft(); // Minimum responses have not been received yet + + if (!adaptiveTimeoutCalculated) { + // Recompute timeout when minimum responses have been received + long timeLeftMs = query.getTimeLeft(); + adaptiveTimeoutMin = (long) (timeLeftMs * config.minWaitAfterCoverageFactor()); + adaptiveTimeoutMax = (long) (timeLeftMs * config.maxWaitAfterCoverageFactor()); + adaptiveTimeoutCalculated = true; + } + long now = timer.milliTime(); + int pendingQueries = askedNodes - answeredNodes; + double missWidth = ((100.0 - minimumCoverage) * askedNodes) / 100.0 - 1.0; + double slopedWait = adaptiveTimeoutMin; + if (pendingQueries > 1 && missWidth > 0.0) { + slopedWait += ((adaptiveTimeoutMax - adaptiveTimeoutMin) * (pendingQueries - 1)) / missWidth; + } + long nextAdaptive = (long) slopedWait; + if (now + nextAdaptive >= deadline) { + return deadline - now; + } + deadline = now + nextAdaptive; + + return nextAdaptive; + } + + @Override + public int reason() { + return DEGRADED_BY_ADAPTIVE_TIMEOUT; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java new file mode 100644 index 00000000000..4e19c8a2e22 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java @@ -0,0 +1,102 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.search.result.Coverage; + +import static com.yahoo.container.handler.Coverage.*; + +/** + * Aggregates coverage as responses are added. + * + * @author baldersheim + */ +public class CoverageAggregator { + private final int askedNodes; + private int answeredNodes = 0; + private int answeredNodesParticipated = 0; + private int failedNodes = 0; + private long answeredDocs = 0; + private long answeredActiveDocs = 0; + private long answeredTargetActiveDocs = 0; + private boolean timedOut = false; + private boolean degradedByMatchPhase = false; + CoverageAggregator(int askedNodes) { + this.askedNodes = askedNodes; + } + CoverageAggregator(CoverageAggregator rhs) { + askedNodes = rhs.askedNodes; + answeredNodes = rhs.answeredNodes; + answeredNodesParticipated = rhs.answeredNodesParticipated; + failedNodes = rhs.failedNodes; + answeredDocs = rhs.answeredDocs; + answeredActiveDocs = rhs.answeredActiveDocs; + answeredTargetActiveDocs = rhs.answeredTargetActiveDocs; + timedOut = rhs.timedOut; + degradedByMatchPhase = rhs.degradedByMatchPhase; + } + void add(Coverage source) { + answeredDocs += source.getDocs(); + answeredActiveDocs += source.getActive(); + answeredTargetActiveDocs += source.getTargetActive(); + answeredNodesParticipated += source.getNodes(); + answeredNodes++; + degradedByMatchPhase |= source.isDegradedByMatchPhase(); + timedOut |= source.isDegradedByTimeout(); + } + public int getAskedNodes() { + return askedNodes; + } + public int getAnsweredNodes() { + return answeredNodes; + } + public boolean hasNoAnswers() { return answeredNodes == 0; } + public void setTimedOut() { timedOut = true; } + public void setFailedNodes(int failedNodes) { + this.failedNodes = failedNodes; + } + + public Coverage createCoverage(TimeoutHandler timeoutHandler, boolean useTargetActiveForCoverageComputation) { + Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); + coverage.setNodesTried(askedNodes); + coverage.setTargetActive(answeredTargetActiveDocs); + coverage.useTargetActiveForCoverageComputation(useTargetActiveForCoverageComputation); + int degradedReason = 0; + if (timedOut) { + degradedReason |= timeoutHandler.reason(); + } + if (degradedByMatchPhase) { + degradedReason |= DEGRADED_BY_MATCH_PHASE; + } + coverage.setDegradedReason(degradedReason); + return coverage; + } + public CoverageAggregator adjustedDegradedCoverage(int redundancy, TimeoutHandler timeoutHandler) { + int askedAndFailed = askedNodes + failedNodes; + if (askedAndFailed == answeredNodesParticipated) { + return this; + } + int notAnswered = askedAndFailed - answeredNodesParticipated; + + if (timeoutHandler.reason() == DEGRADED_BY_ADAPTIVE_TIMEOUT) { + CoverageAggregator clone = new CoverageAggregator(this); + return clone.adjustActiveDocs(notAnswered); + } else { + if (askedAndFailed > answeredNodesParticipated) { + CoverageAggregator clone = new CoverageAggregator(this); + int missingNodes = notAnswered - (redundancy - 1); + if (missingNodes > 0) { + clone.adjustActiveDocs(missingNodes); + } + clone.timedOut = true; + return clone; + } + } + return this; + } + private CoverageAggregator adjustActiveDocs(int numMissingNodes) { + if (answeredNodesParticipated > 0) { + answeredActiveDocs += (numMissingNodes * answeredActiveDocs / answeredNodesParticipated); + answeredTargetActiveDocs += (numMissingNodes * answeredTargetActiveDocs / answeredNodesParticipated); + } + return this; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 8c92e8b0270..2717b48839e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.SearchCluster; -import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -25,10 +25,6 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_ADAPTIVE_TIMEOUT; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; - /** * InterleavedSearchInvoker uses multiple {@link SearchInvoker} objects to interface with content * nodes in parallel. Operationally it first sends requests to all contained invokers and then @@ -40,38 +36,35 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private static final Logger log = Logger.getLogger(InterleavedSearchInvoker.class.getName()); + private final Timer timer; private final Set<SearchInvoker> invokers; private final SearchCluster searchCluster; private final Group group; private final LinkedBlockingQueue<SearchInvoker> availableForProcessing; private final Set<Integer> alreadyFailedNodes; + private final CoverageAggregator coverageAggregator; private Query query; - private boolean adaptiveTimeoutCalculated = false; - private long adaptiveTimeoutMin = 0; - private long adaptiveTimeoutMax = 0; - private long deadline = 0; - - private long answeredDocs = 0; - private long answeredActiveDocs = 0; - private long answeredTargetActiveDocs = 0; - private int askedNodes = 0; - private int answeredNodes = 0; - private int answeredNodesParticipated = 0; - private boolean timedOut = false; - private boolean degradedByMatchPhase = false; - - public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, + private TimeoutHandler timeoutHandler; + public InterleavedSearchInvoker(Timer timer, Collection<SearchInvoker> invokers, SearchCluster searchCluster, Group group, Set<Integer> alreadyFailedNodes) { super(Optional.empty()); + this.timer = timer; this.invokers = Collections.newSetFromMap(new IdentityHashMap<>()); this.invokers.addAll(invokers); this.searchCluster = searchCluster; this.group = group; this.availableForProcessing = newQueue(); this.alreadyFailedNodes = alreadyFailedNodes; + coverageAggregator = new CoverageAggregator(invokers.size()); + } + + private TimeoutHandler createTimeoutHandler(DispatchConfig config, int askedNodes, Query query) { + return (config.minSearchCoverage() < 100.0D) + ? new AdaptiveTimeoutHandler(timer, config, askedNodes, query) + : new SimpleTimeoutHandler(query); } /** @@ -83,7 +76,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM protected Object sendSearchRequest(Query query, Object unusedContext) throws IOException { this.query = query; invokers.forEach(invoker -> invoker.setMonitor(this)); - deadline = currentTime() + query.getTimeLeft(); int originalHits = query.getHits(); int originalOffset = query.getOffset(); @@ -101,8 +93,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM Object context = null; for (SearchInvoker invoker : invokers) { context = invoker.sendSearchRequest(query, context); - askedNodes++; } + timeoutHandler = createTimeoutHandler(searchCluster.dispatchConfig(), invokers.size(), query); query.setHits(originalHits); query.setOffset(originalOffset); @@ -119,14 +111,15 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM while (!invokers.isEmpty() && nextTimeout >= 0) { SearchInvoker invoker = availableForProcessing.poll(nextTimeout, TimeUnit.MILLISECONDS); if (invoker == null) { - log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); + log.fine(() -> "Search timed out with " + coverageAggregator.getAskedNodes() + " requests made, " + + coverageAggregator.getAnsweredNodes() + " responses received"); break; } else { InvokerResult toMerge = invoker.getSearchResult(execution); merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } - nextTimeout = nextTimeout(); + nextTimeout = timeoutHandler.nextTimeoutMS(coverageAggregator.getAnsweredNodes()); } } catch (InterruptedException e) { throw new RuntimeException("Interrupted while waiting for search results", e); @@ -134,7 +127,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM groupingResultAggregator.toAggregatedHit().ifPresent(h -> result.getResult().hits().add(h)); insertNetworkErrors(result.getResult()); - result.getResult().setCoverage(createCoverage()); + CoverageAggregator adjusted = coverageAggregator.adjustedDegradedCoverage(redundancyForCoverage(searchCluster.dispatchConfig()), timeoutHandler); + result.getResult().setCoverage(adjusted.createCoverage(timeoutHandler, searchCluster.dispatchConfig().computeCoverageFromTargetActiveDocs())); int needed = query.getOffset() + query.getHits(); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { @@ -144,9 +138,13 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return result; } + private int redundancyForCoverage(DispatchConfig config) { + return (int)(config.computeCoverageFromTargetActiveDocs() ? config.redundancy() : config.searchableCopies()); + } + private void insertNetworkErrors(Result result) { // Network errors will be reported as errors only when all nodes fail, otherwise they are just traced - boolean asErrors = answeredNodes == 0; + boolean asErrors = coverageAggregator.hasNoAnswers(); if (!invokers.isEmpty()) { String keys = invokers.stream().map(SearchInvoker::distributionKey).map(dk -> dk.map(i -> i.toString()).orElse("(unspecified)")) @@ -158,7 +156,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } else { query.trace("Backend communication timeout on nodes with distribution-keys: " + keys, 2); } - timedOut = true; + coverageAggregator.setTimedOut(); } if (alreadyFailedNodes != null) { var message = "Connection failure on nodes with distribution-keys: " @@ -168,51 +166,13 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } else { query.trace(message, 2); } - int failed = alreadyFailedNodes.size(); - askedNodes += failed; - answeredNodes += failed; - } - } - - private long nextTimeout() { - DispatchConfig config = searchCluster.dispatchConfig(); - double minimumCoverage = config.minSearchCoverage(); - - if (askedNodes == answeredNodes || minimumCoverage >= 100.0) { - return query.getTimeLeft(); - } - int minimumResponses = (int) Math.ceil(askedNodes * minimumCoverage / 100.0); - - if (answeredNodes < minimumResponses) { - return query.getTimeLeft(); - } - - long timeLeft = query.getTimeLeft(); - if (!adaptiveTimeoutCalculated) { - adaptiveTimeoutMin = (long) (timeLeft * config.minWaitAfterCoverageFactor()); - adaptiveTimeoutMax = (long) (timeLeft * config.maxWaitAfterCoverageFactor()); - adaptiveTimeoutCalculated = true; - } - - long now = currentTime(); - int pendingQueries = askedNodes - answeredNodes; - double missWidth = ((100.0 - config.minSearchCoverage()) * askedNodes) / 100.0 - 1.0; - double slopedWait = adaptiveTimeoutMin; - if (pendingQueries > 1 && missWidth > 0.0) { - slopedWait += ((adaptiveTimeoutMax - adaptiveTimeoutMin) * (pendingQueries - 1)) / missWidth; - } - long nextAdaptive = (long) slopedWait; - if (now + nextAdaptive >= deadline) { - return deadline - now; + coverageAggregator.setFailedNodes(alreadyFailedNodes.size()); } - deadline = now + nextAdaptive; - - return nextAdaptive; } private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current, GroupingResultAggregator groupingResultAggregator) { - collectCoverage(partialResult.getResult().getCoverage(true)); + coverageAggregator.add(partialResult.getResult().getCoverage(true)); result.mergeWith(partialResult.getResult()); List<Hit> partialNonLean = partialResult.getResult().hits().asUnorderedHits(); @@ -265,55 +225,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return merged; } - private void collectCoverage(Coverage source) { - answeredDocs += source.getDocs(); - answeredActiveDocs += source.getActive(); - answeredTargetActiveDocs += source.getTargetActive(); - answeredNodesParticipated += source.getNodes(); - answeredNodes++; - degradedByMatchPhase |= source.isDegradedByMatchPhase(); - timedOut |= source.isDegradedByTimeout(); - } - - private Coverage createCoverage() { - adjustDegradedCoverage(); - - Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); - coverage.setNodesTried(askedNodes); - coverage.setTargetActive(answeredTargetActiveDocs); - int degradedReason = 0; - if (timedOut) { - degradedReason |= (adaptiveTimeoutCalculated ? DEGRADED_BY_ADAPTIVE_TIMEOUT : DEGRADED_BY_TIMEOUT); - } - if (degradedByMatchPhase) { - degradedReason |= DEGRADED_BY_MATCH_PHASE; - } - coverage.setDegradedReason(degradedReason); - return coverage; - } - - private void adjustDegradedCoverage() { - if (askedNodes == answeredNodesParticipated) { - return; - } - int notAnswered = askedNodes - answeredNodesParticipated; - - if (adaptiveTimeoutCalculated && answeredNodesParticipated > 0) { - answeredActiveDocs += (notAnswered * answeredActiveDocs / answeredNodesParticipated); - answeredTargetActiveDocs += (notAnswered * answeredTargetActiveDocs / answeredNodesParticipated); - } else { - if (askedNodes > answeredNodesParticipated) { - int searchableCopies = (int) searchCluster.dispatchConfig().searchableCopies(); - int missingNodes = notAnswered - (searchableCopies - 1); - if (answeredNodesParticipated > 0) { - answeredActiveDocs += (missingNodes * answeredActiveDocs / answeredNodesParticipated); - answeredTargetActiveDocs += (missingNodes * answeredTargetActiveDocs / answeredNodesParticipated); - timedOut = true; - } - } - } - } - private void ejectInvoker(SearchInvoker invoker) { invokers.remove(invoker); invoker.release(); @@ -340,11 +251,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } // For overriding in tests - protected long currentTime() { - return System.currentTimeMillis(); - } - - // For overriding in tests protected LinkedBlockingQueue<SearchInvoker> newQueue() { return new LinkedBlockingQueue<>(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index eb379a51ed4..02cf11c9fe7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -89,7 +90,7 @@ public abstract class InvokerFactory { if (invokers.size() == 1 && failed == null) { return Optional.of(invokers.get(0)); } else { - return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster, group, failed)); + return Optional.of(new InterleavedSearchInvoker(Timer.monotonic, invokers, searchCluster, group, failed)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java new file mode 100644 index 00000000000..90b6bf87a0b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java @@ -0,0 +1,26 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.search.Query; + +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; + +/** + * Computes the timeout based solely on the query timeout + * + * @author baldersheim + */ +public class SimpleTimeoutHandler implements TimeoutHandler { + private final Query query; + SimpleTimeoutHandler(Query query) { + this.query = query; + } + @Override + public long nextTimeoutMS(int answeredNodes) { + return query.getTimeLeft(); + } + + @Override + public int reason() { + return DEGRADED_BY_TIMEOUT; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java new file mode 100644 index 00000000000..a8ac0d14ddc --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java @@ -0,0 +1,15 @@ +package com.yahoo.search.dispatch; + +/** + * Computes next timeout in milliseconds + * + * @author baldersheim + */ +public interface TimeoutHandler { + long nextTimeoutMS(int answeredNodes); + + /** + * Return a bitmask from com.yahoo.container.handler.Coverage.DEGRADED.... set + */ + int reason(); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 82d3d98d9ef..25617e48aa1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -307,6 +307,7 @@ public class SearchCluster implements NodeManager<Node> { if (fullCoverage) { log.info("Cluster " + clusterId + ": " + group + " has full coverage. " + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size()); } else { StringBuilder unresponsive = new StringBuilder(); @@ -316,6 +317,7 @@ public class SearchCluster implements NodeManager<Node> { } log.warning("Cluster " + clusterId + ": " + group + " has reduced coverage: " + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index bd72ed9fd08..d374bfdeb7b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.data.access.simple.Value; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -40,7 +41,7 @@ public class JuniperSearcherTestCase { * @param sdName the search definition type of the returned hit * @param content the content of the "dynteaser" field of the returned hit */ - private Chain<Searcher> createSearchChain(String sdName, String content) { + private Chain<Searcher> createSearchChain(String sdName, Object content) { JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), new QrSearchersConfig(new QrSearchersConfig.Builder())); @@ -50,7 +51,7 @@ public class JuniperSearcherTestCase { return new Chain<>(searcher, docsource); } - private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + private void addResult(Query query, String sdName, Object content, DocumentSourceSearcher docsource) { Result r = new Result(query); FastHit hit = new FastHit(); hit.setId("http://abc.html"); @@ -62,7 +63,7 @@ public class JuniperSearcherTestCase { } /** Creates a result of the search definiton "one" */ - private Result createResult(String content) { + private Result createResult(Object content) { return createResult("one", content, true); } @@ -70,7 +71,7 @@ public class JuniperSearcherTestCase { return createResult(sdName, content, true); } - private Result createResult(String sdName, String content, boolean bolding) { + private Result createResult(String sdName, Object content, boolean bolding) { Chain<Searcher> chain = createSearchChain(sdName, content); Query query = new Query("?query=12"); if ( ! bolding) @@ -123,6 +124,17 @@ public class JuniperSearcherTestCase { } @Test + void test_field_rewriting_for_array_of_string_field() { + var content = new Value.ArrayValue() + .add("\u001Faaa\u001F\u001Ebbb\u001Fccc\u001Fddd") + .add("\u001Feee\u001F\u001Efff\u001Fggg\u001Fhhh"); + Result check = createResult(content); + assertEquals(1, check.getHitCount()); + assertEquals("[\"<hi>aaa</hi><sep />bbb<hi>ccc</hi>ddd\",\"<hi>eee</hi><sep />fff<hi>ggg</hi>hhh\"]", + check.hits().get(0).getField("dynteaser").toString()); + } + + @Test void testNoRewritingDueToSearchDefinition() { Result check = createResult("two", "\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&"); assertEquals(1, check.getHitCount()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index a88046197e0..180beb43ee8 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.document.GlobalId; import com.yahoo.document.idstring.IdString; import com.yahoo.prelude.fastsearch.FastHit; @@ -20,6 +21,7 @@ import com.yahoo.searchlib.aggregation.MinAggregationResult; import com.yahoo.searchlib.expression.IntegerResultNode; import com.yahoo.searchlib.expression.StringResultNode; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -53,52 +55,55 @@ public class InterleavedSearchInvokerTest { @Test void requireThatAdaptiveTimeoutsAreNotUsedWithFullCoverageRequirement() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3)) { - expectedEvents.add(new Event(5000, 100, 0)); - expectedEvents.add(new Event(4900, 100, 1)); - expectedEvents.add(new Event(4800, 100, 2)); + expectedEvents.add(new Event(5000, 100, 0)); + expectedEvents.add(new Event(4900, 100, 1)); + expectedEvents.add(new Event(4800, 100, 2)); - invoker.search(query, null); + invoker.search(query, null); - assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); + assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); + } } @Test void requireThatTimeoutsAreNotMarkedAsAdaptive() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3)) { - expectedEvents.add(new Event(5000, 300, 0)); - expectedEvents.add(new Event(4700, 300, 1)); - expectedEvents.add(null); + expectedEvents.add(new Event(5000, 300, 0)); + expectedEvents.add(new Event(4700, 300, 1)); + expectedEvents.add(null); - Result result = invoker.search(query, null); + Result result = invoker.search(query, null); - assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); - assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); - var message = findTrace(result, "Backend communication timeout"); - assertTrue(message.isPresent(), "Timeout should be reported in a trace message"); - assertTrue(result.getCoverage(false).isDegradedByTimeout(), "Degradation reason is a normal timeout"); + assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); + assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); + var message = findTrace(result, "Backend communication timeout"); + assertTrue(message.isPresent(), "Timeout should be reported in a trace message"); + assertTrue(result.getCoverage(false).isDegradedByTimeout(), "Degradation reason is a normal timeout"); + } } @Test void requireThatAdaptiveTimeoutDecreasesTimeoutWhenCoverageIsReached() throws IOException { SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(50.0), 1, 4); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 4); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 4)) { - expectedEvents.add(new Event(5000, 100, 0)); - expectedEvents.add(new Event(4900, 100, 1)); - expectedEvents.add(new Event(2400, 100, 2)); - expectedEvents.add(new Event(0, 0, null)); + expectedEvents.add(new Event(5000, 100, 0)); + expectedEvents.add(new Event(4900, 100, 1)); + expectedEvents.add(new Event(2400, 100, 2)); + expectedEvents.add(new Event(0, 0, null)); - Result result = invoker.search(query, null); + Result result = invoker.search(query, null); - assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); - assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); - var message = findTrace(result, "Backend communication timeout"); - assertTrue(message.isPresent(), "Timeout should be reported in a trace message"); - assertTrue(result.getCoverage(false).isDegradedByAdapativeTimeout(), "Degradataion reason is an adaptive timeout"); + assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); + assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); + var message = findTrace(result, "Backend communication timeout"); + assertTrue(message.isPresent(), "Timeout should be reported in a trace message"); + assertTrue(result.getCoverage(false).isDegradedByAdapativeTimeout(), "Degradataion reason is an adaptive timeout"); + } } @Test @@ -106,20 +111,21 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0)) { - expectedEvents.add(new Event(null, 100, 0)); - expectedEvents.add(new Event(null, 200, 1)); + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query, null); - Coverage cov = result.getCoverage(true); - assertEquals(100000L, cov.getDocs()); - assertEquals(2, cov.getNodes()); - assertTrue(cov.getFull()); - assertEquals(100, cov.getResultPercentage()); - assertEquals(1, cov.getResultSets()); - assertEquals(1, cov.getFullResultSets()); + Coverage cov = result.getCoverage(true); + assertEquals(100000L, cov.getDocs()); + assertEquals(2, cov.getNodes()); + assertTrue(cov.getFull()); + assertEquals(100, cov.getResultPercentage()); + assertEquals(1, cov.getResultSets()); + assertEquals(1, cov.getFullResultSets()); + } } @Test @@ -127,21 +133,22 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(10101, 50155, 50155, 1, 1, DEGRADED_BY_MATCH_PHASE))); invokers.add(new MockInvoker(1, createCoverage(13319, 49845, 49845, 1, 1, DEGRADED_BY_MATCH_PHASE))); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0)) { - expectedEvents.add(new Event(null, 100, 0)); - expectedEvents.add(new Event(null, 200, 1)); + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query, null); - Coverage cov = result.getCoverage(true); - assertEquals(23420L, cov.getDocs()); - assertEquals(2, cov.getNodes()); - assertFalse(cov.getFull()); - assertEquals(23, cov.getResultPercentage()); - assertEquals(1, cov.getResultSets()); - assertEquals(0, cov.getFullResultSets()); - assertTrue(cov.isDegradedByMatchPhase()); + Coverage cov = result.getCoverage(true); + assertEquals(23420L, cov.getDocs()); + assertEquals(2, cov.getNodes()); + assertFalse(cov.getFull()); + assertEquals(23, cov.getResultPercentage()); + assertEquals(1, cov.getResultSets()); + assertEquals(0, cov.getFullResultSets()); + assertTrue(cov.isDegradedByMatchPhase()); + } } @Test @@ -149,21 +156,22 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(5000, 50155, 50155, 1, 1, DEGRADED_BY_TIMEOUT))); invokers.add(new MockInvoker(1, createCoverage(4900, 49845, 49845, 1, 1, DEGRADED_BY_TIMEOUT))); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0)) { - expectedEvents.add(new Event(null, 100, 0)); - expectedEvents.add(new Event(null, 200, 1)); + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query, null); - Coverage cov = result.getCoverage(true); - assertEquals(9900L, cov.getDocs()); - assertEquals(2, cov.getNodes()); - assertFalse(cov.getFull()); - assertEquals(10, cov.getResultPercentage()); - assertEquals(1, cov.getResultSets()); - assertEquals(0, cov.getFullResultSets()); - assertTrue(cov.isDegradedByTimeout()); + Coverage cov = result.getCoverage(true); + assertEquals(9900L, cov.getDocs()); + assertEquals(2, cov.getNodes()); + assertFalse(cov.getFull()); + assertEquals(10, cov.getResultPercentage()); + assertEquals(1, cov.getResultSets()); + assertEquals(0, cov.getFullResultSets()); + assertTrue(cov.isDegradedByTimeout()); + } } @Test @@ -171,22 +179,23 @@ public class InterleavedSearchInvokerTest { SearchCluster cluster = new MockSearchCluster("!", 1, 2); invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); - - expectedEvents.add(new Event(null, 100, 0)); - expectedEvents.add(null); - - Result result = invoker.search(query, null); - - Coverage cov = result.getCoverage(true); - assertEquals(50155L, cov.getDocs()); - assertEquals(1, cov.getNodes()); - assertEquals(2, cov.getNodesTried()); - assertFalse(cov.getFull()); - assertEquals(50, cov.getResultPercentage()); - assertEquals(1, cov.getResultSets()); - assertEquals(0, cov.getFullResultSets()); - assertTrue(cov.isDegradedByTimeout()); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0)) { + + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(null); + + Result result = invoker.search(query, null); + + Coverage cov = result.getCoverage(true); + assertEquals(50155L, cov.getDocs()); + assertEquals(1, cov.getNodes()); + assertEquals(2, cov.getNodesTried()); + assertFalse(cov.getFull()); + assertEquals(50, cov.getResultPercentage()); + assertEquals(1, cov.getResultSets()); + assertEquals(0, cov.getFullResultSets()); + assertTrue(cov.isDegradedByTimeout()); + } } static class MetaHit extends Hit { @@ -206,24 +215,25 @@ public class InterleavedSearchInvokerTest { private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK, Group group) throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, group); - query.setHits(8); - query.properties().set(Dispatcher.topKProbability, topKProbability); - SearchInvoker [] invokers = invoker.invokers().toArray(new SearchInvoker[0]); - Result result = invoker.search(query, null); - assertEquals(2, invokers.length); - assertEquals(expectedK, ((MockInvoker)invokers[0]).hitsRequested); - assertEquals(8, result.hits().size()); - assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(9.0, result.hits().get(1).getRelevance().getScore(), DELTA); - assertEquals(8.5, result.hits().get(2).getRelevance().getScore(), DELTA); - assertEquals(8.0, result.hits().get(3).getRelevance().getScore(), DELTA); - assertEquals(7.5, result.hits().get(4).getRelevance().getScore(), DELTA); - assertEquals(7.0, result.hits().get(5).getRelevance().getScore(), DELTA); - assertEquals(6.0, result.hits().get(6).getRelevance().getScore(), DELTA); - assertEquals(3.0, result.hits().get(7).getRelevance().getScore(), DELTA); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(8, result.getQuery().getHits()); + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, group)) { + query.setHits(8); + query.properties().set(Dispatcher.topKProbability, topKProbability); + SearchInvoker[] invokers = invoker.invokers().toArray(new SearchInvoker[0]); + Result result = invoker.search(query, null); + assertEquals(2, invokers.length); + assertEquals(expectedK, ((MockInvoker) invokers[0]).hitsRequested); + assertEquals(8, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(9.0, result.hits().get(1).getRelevance().getScore(), DELTA); + assertEquals(8.5, result.hits().get(2).getRelevance().getScore(), DELTA); + assertEquals(8.0, result.hits().get(3).getRelevance().getScore(), DELTA); + assertEquals(7.5, result.hits().get(4).getRelevance().getScore(), DELTA); + assertEquals(7.0, result.hits().get(5).getRelevance().getScore(), DELTA); + assertEquals(6.0, result.hits().get(6).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(7).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(8, result.getQuery().getHits()); + } } @Test @@ -258,68 +268,74 @@ public class InterleavedSearchInvokerTest { @Test void requireThatMergeOfConcreteHitsObeySorting() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of())); - query.setHits(12); - Result result = invoker.search(query, null); - assertEquals(10, result.hits().size()); - assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(12, result.getQuery().getHits()); - - invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of())); - result = invoker.search(query, null); - assertEquals(10, result.hits().size()); - assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(12, result.getQuery().getHits()); + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of()))) { + query.setHits(12); + Result result = invoker.search(query, null); + assertEquals(10, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(12, result.getQuery().getHits()); + } + + try ( InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of()))) { + Result result = invoker.search(query, null); + assertEquals(10, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(12, result.getQuery().getHits()); + } } @Test void requireThatMergeOfConcreteHitsObeyOffset() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of())); - query.setHits(3); - query.setOffset(5); - Result result = invoker.search(query, null); - assertEquals(3, result.hits().size()); - assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(3, result.getQuery().getHits()); - - invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of())); - query.setOffset(5); - result = invoker.search(query, null); - assertEquals(3, result.hits().size()); - assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(3, result.getQuery().getHits()); + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of()))) { + query.setHits(3); + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(3, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(3, result.getQuery().getHits()); + } + + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of()))) { + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(3, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(3, result.getQuery().getHits()); + } } @Test void requireThatMergeOfConcreteHitsObeyOffsetWithAuxilliaryStuff() throws IOException { - InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux, new Group(0, List.of())); - query.setHits(3); - query.setOffset(5); - Result result = invoker.search(query, null); - assertEquals(7, result.hits().size()); - assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); - assertTrue(result.hits().get(3) instanceof MetaHit); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(3, result.getQuery().getHits()); - - invoker = createInterLeavedTestInvoker(B5Aux, A5Aux, new Group(0, List.of())); - query.setOffset(5); - result = invoker.search(query, null); - assertEquals(7, result.hits().size()); - assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); - assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); - assertTrue(result.hits().get(3) instanceof MetaHit); - assertEquals(0, result.getQuery().getOffset()); - assertEquals(3, result.getQuery().getHits()); + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux, new Group(0, List.of()))) { + query.setHits(3); + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(7, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertTrue(result.hits().get(3) instanceof MetaHit); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(3, result.getQuery().getHits()); + } + + try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5Aux, A5Aux, new Group(0, List.of()))) { + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(7, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertTrue(result.hits().get(3) instanceof MetaHit); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(3, result.getQuery().getHits()); + } } @Test @@ -347,12 +363,15 @@ public class InterleavedSearchInvokerTest { .addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3)))); invokers.add(new MockInvoker(0).setHits(List.of(new GroupingListHit(List.of(grouping2))))); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, new Group(0, List.of()), Collections.emptySet()); - invoker.responseAvailable(invokers.get(0)); - invoker.responseAvailable(invokers.get(1)); - Result result = invoker.search(query, null); - assertEquals(1, ((GroupingListHit) result.hits().get(0)).getGroupingList().size()); - + try (InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, cluster, new Group(0, List.of()), Collections.emptySet())) { + invoker.responseAvailable(invokers.get(0)); + invoker.responseAvailable(invokers.get(1)); + Result result = invoker.search(query, null); + assertEquals(1, ((GroupingListHit) result.hits().get(0)).getGroupingList().size()); + } + for (SearchInvoker invoker : invokers) { + invoker.close(); + } } private static InterleavedSearchInvoker createInterLeavedTestInvoker(List<Double> a, List<Double> b, Group group) { @@ -360,7 +379,7 @@ public class InterleavedSearchInvokerTest { List<SearchInvoker> invokers = new ArrayList<>(); invokers.add(createInvoker(a, 0)); invokers.add(createInvoker(b, 1)); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, group, Collections.emptySet()); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, cluster, group, Collections.emptySet()); invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); return invoker; @@ -381,29 +400,46 @@ public class InterleavedSearchInvokerTest { return hits; } - @Test - void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpected() throws IOException { - SearchCluster cluster = new MockSearchCluster("!", 1, 2); - invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); + void verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(DispatchConfig dispatchConfig, int expectedCoverage) throws IOException { + SearchCluster cluster = new MockSearchCluster("!", dispatchConfig, 1, 2); + invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 60000, 1, 1, 0))); Coverage errorCoverage = new Coverage(0, 0, 0); errorCoverage.setNodesTried(1); invokers.add(new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError("node is down"), errorCoverage)); - SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0); - - expectedEvents.add(new Event(null, 1, 1)); - expectedEvents.add(new Event(null, 100, 0)); - - Result result = invoker.search(query, null); - - Coverage cov = result.getCoverage(true); - assertEquals(50155L, cov.getDocs()); - assertEquals(1, cov.getNodes()); - assertEquals(2, cov.getNodesTried()); - assertFalse(cov.getFull()); - assertEquals(50, cov.getResultPercentage()); - assertEquals(1, cov.getResultSets()); - assertEquals(0, cov.getFullResultSets()); - assertTrue(cov.isDegradedByTimeout()); + try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 0)) { + + expectedEvents.add(new Event(null, 1, 1)); + expectedEvents.add(new Event(null, 100, 0)); + + Result result = invoker.search(query, null); + + Coverage cov = result.getCoverage(true); + assertEquals(50155L, cov.getDocs()); + assertEquals(1, cov.getNodes()); + assertEquals(2, cov.getNodesTried()); + assertFalse(cov.getFull()); + assertEquals(expectedCoverage, cov.getResultPercentage()); + assertEquals(1, cov.getResultSets()); + assertEquals(0, cov.getFullResultSets()); + assertTrue(cov.isDegradedByTimeout()); + } + } + @Test + void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpectedUsingActiveDocs() throws IOException { + verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(MockSearchCluster.createDispatchConfig(100.0, List.of()) + .searchableCopies(1) + .redundancy(2) + .build(), + 50); + } + @Test + void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpectedUsingTargetActiveDocs() throws IOException { + verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(MockSearchCluster.createDispatchConfig(100.0, List.of()) + .searchableCopies(1) + .redundancy(1) + .computeCoverageFromTargetActiveDocs(true) + .build(), + 42); } private InterleavedSearchInvoker createInterleavedInvoker(SearchCluster searchCluster, Group group, int numInvokers) { @@ -411,12 +447,7 @@ public class InterleavedSearchInvokerTest { invokers.add(new MockInvoker(i)); } - return new InterleavedSearchInvoker(invokers, searchCluster, group,null) { - - @Override - protected long currentTime() { - return clock.millis(); - } + return new InterleavedSearchInvoker(Timer.wrap(clock), invokers, searchCluster, group,null) { @Override protected LinkedBlockingQueue<SearchInvoker> newQueue() { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index a076b929ad7..7a11e906293 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -105,14 +105,14 @@ public class MockSearchCluster extends SearchCluster { return createDispatchConfig(100.0, nodes); } public static DispatchConfig createDispatchConfig(List<Node> nodes) { - return createDispatchConfig(100.0, nodes); + return createDispatchConfig(100.0, nodes).build(); } public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) { - return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)); + return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)).build(); } - public static DispatchConfig createDispatchConfig(double minSearchCoverage, List<Node> nodes) { + public static DispatchConfig.Builder createDispatchConfig(double minSearchCoverage, List<Node> nodes) { DispatchConfig.Builder builder = new DispatchConfig.Builder(); builder.minActivedocsPercentage(88.0); builder.minSearchCoverage(minSearchCoverage); @@ -125,7 +125,7 @@ public class MockSearchCluster extends SearchCluster { for (Node n : nodes) { builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group())); } - return new DispatchConfig(builder); + return builder; } } diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java index f59a7a94f0a..e2675b09307 100644 --- a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java @@ -1,9 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result.test; +package com.yahoo.search.result; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.result.Coverage; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,6 +33,17 @@ public class CoverageTestCase { } @Test + void testCoverageBasedOnActive() { + var c = new Coverage(8, 10).setTargetActive(16); + assertEquals(80, c.getResultPercentage()); + } + @Test + void testCoverageBasedOnTargetActive() { + var c = new Coverage(8, 10).setTargetActive(16).useTargetActiveForCoverageComputation(true); + assertEquals(50, c.getResultPercentage()); + } + + @Test void testDefaultCoverage() { boolean create = true; @@ -67,10 +77,7 @@ public class CoverageTestCase { assertEquals(1, federationSearcherResult.getCoverage(create).getResultSets()); } - @Test - void testCoverageConversion() { - Coverage c = new Coverage(6, 10); - c.setDegradedReason(7); + void verifyCoverageConversion(com.yahoo.container.handler.Coverage c) { com.yahoo.container.logging.Coverage lc = c.toLoggingCoverage(); assertEquals(lc.getDocs(), c.getDocs()); assertEquals(lc.getActive(), c.getActive()); @@ -83,4 +90,10 @@ public class CoverageTestCase { assertEquals(lc.isDegradedByTimeout(), c.isDegradedByTimeout()); } + @Test + void testCoverageConversion() { + verifyCoverageConversion(new Coverage(6, 10).setDegradedReason(7).setTargetActive(12).useTargetActiveForCoverageComputation(false)); + verifyCoverageConversion(new Coverage(6, 10).setDegradedReason(7).setTargetActive(12).useTargetActiveForCoverageComputation(true)); + } + } |