diff options
3 files changed, 161 insertions, 11 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 76157175c3d..c636d4e4b22 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.IdentityHashMap; +import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; @@ -95,6 +96,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM protected Result getSearchResult(Execution execution) throws IOException { Result result = new Result(query); List<Hit> merged = Collections.emptyList(); + List<Hit> auxiliary = new ArrayList<>(); long nextTimeout = query.getTimeLeft(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { @@ -103,7 +105,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); break; } else { - merged = mergeResult(result, invoker.getSearchResult(execution), merged); + merged = mergeResult(result, invoker.getSearchResult(execution), merged, auxiliary); ejectInvoker(invoker); } nextTimeout = nextTimeout(); @@ -115,6 +117,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM insertNetworkErrors(result); result.setCoverage(createCoverage()); int needed = query.getOffset() + query.getHits(); + result.hits().addAll(auxiliary); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { result.hits().add(merged.get(index)); } @@ -187,13 +190,21 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private List<Hit> mergeResult(Result result, Result partialResult, List<Hit> current) { + private List<Hit> mergeResult(Result result, Result partialResult, List<Hit> current, List<Hit> auxiliaryHits) { collectCoverage(partialResult.getCoverage(true)); result.mergeWith(partialResult); List<Hit> partial = partialResult.hits().asUnorderedHits(); if (current.isEmpty() ) { - return partial; + boolean hasAuxillary = false; + for(Hit hit : partial) { + if (hit.isAuxiliary()) { + hasAuxillary = true; + break; + } + } + if ( ! hasAuxillary) + return partial; } if (partial.isEmpty()) { return current; @@ -204,21 +215,47 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM int indexCurrent = 0; int indexPartial = 0; while (indexCurrent < current.size() && indexPartial < partial.size() && merged.size() < needed) { - int cmpRes = current.get(indexCurrent).compareTo(partial.get(indexPartial)); + Hit incommingHit = partial.get(indexPartial); + if (incommingHit.isAuxiliary()) { + auxiliaryHits.add(incommingHit); + indexPartial++; + continue; + } + Hit currentHit = current.get(indexCurrent); + if (currentHit.isAuxiliary()) { + auxiliaryHits.add(currentHit); + indexCurrent++; + continue; + } + + int cmpRes = currentHit.compareTo(incommingHit); if (cmpRes < 0) { - merged.add(current.get(indexCurrent++)); + merged.add(currentHit); + indexCurrent++; } else if (cmpRes > 0) { - merged.add(partial.get(indexPartial++)); + merged.add(incommingHit); + indexPartial++; } else { // Duplicates - merged.add(current.get(indexCurrent++)); + merged.add(currentHit); + indexCurrent++; indexPartial++; } } - while ((indexCurrent < current.size()) && (merged.size() < needed)) { - merged.add(current.get(indexCurrent++)); + while (indexCurrent < current.size()) { + Hit h = current.get(indexCurrent++); + if (h.isAuxiliary()) { + auxiliaryHits.add(h); + } else if (merged.size() < needed) { + merged.add(h); + } } - while ((indexPartial < partial.size()) && (merged.size() < needed)) { - merged.add(partial.get(indexPartial++)); + while (indexPartial < partial.size()) { + Hit h = partial.get(indexPartial++); + if (h.isAuxiliary()) { + auxiliaryHits.add(h); + } else if (merged.size() < needed) { + merged.add(h); + } } return merged; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 8686ddf229b..2306a395c57 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -1,11 +1,18 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.DefaultErrorHit; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; import com.yahoo.test.ManualClock; import org.junit.Test; @@ -13,6 +20,8 @@ import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -179,6 +188,99 @@ public class InterleavedSearchInvokerTest { assertThat(cov.isDegradedByTimeout(), is(true)); } + static class MetaHit extends Hit { + MetaHit(Double relevance) { + super(new Relevance(relevance)); + } + @Override + public boolean isMeta() { + return true; + } + } + + private static final double DELTA = 0.000000000001; + private static final List<Double> A5 = Arrays.asList(11.0,8.5,7.5,3.0,2.0); + private static final List<Double> B5 = Arrays.asList(9.0,8.0,7.0,6.0,1.0); + private static final List<Double> A5Aux = Arrays.asList(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0); + private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); + + @Test + public void requireThatMergeOfConcreteHitsObeySorting() throws IOException { + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5); + Result result = invoker.search(query, null); + assertEquals(10, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); + + invoker = createInterLeavedTestInvoker(B5, A5); + result = invoker.search(query, null); + assertEquals(10, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); + } + + @Test + public void requireThatMergeOfConcreteHitsObeyOffset() throws IOException { + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5); + query.setHits(3); + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(3, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + + invoker = createInterLeavedTestInvoker(B5, A5); + result = invoker.search(query, null); + assertEquals(3, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + } + + @Test + public void requireThatMergeOfConcreteHitsObeyOffsetWithAuxilliaryStuff() throws IOException { + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux); + query.setHits(3); + query.setOffset(5); + Result result = invoker.search(query, null); + assertEquals(7, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertTrue(result.hits().get(3) instanceof MetaHit); + + invoker = createInterLeavedTestInvoker(B5Aux, A5Aux); + result = invoker.search(query, null); + assertEquals(7, result.hits().size()); + assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); + assertTrue(result.hits().get(3) instanceof MetaHit); + } + + private static InterleavedSearchInvoker createInterLeavedTestInvoker(List<Double> a, List<Double> b) { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + List<SearchInvoker> invokers = new ArrayList<>(); + invokers.add(createInvoker(a, 0)); + invokers.add(createInvoker(b, 1)); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, Collections.emptySet()); + invoker.responseAvailable(invokers.get(0)); + invoker.responseAvailable(invokers.get(1)); + return invoker; + } + private static MockInvoker createInvoker(List<Double> scores, int distributionKey) { + return new MockInvoker(0).setHits(createHits(scores, distributionKey, distributionKey)); + } + + private static List<Hit> createHits(List<Double> scores, int partId, int distributionKey) { + List<Hit> hits= new ArrayList<>(scores.size()); + for (Double value : scores) { + if (value < 0) { + hits.add(new MetaHit(value)); + } else { + hits.add(new FastHit(new GlobalId(IdString.createIdString("id:test:test::" + value)), new Relevance(value), partId, distributionKey)); + } + } + return hits; + } + @Test public void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpected() throws IOException { SearchCluster cluster = new MockSearchCluster("!", 1, 2); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index e347a884c17..4b2f63d6b89 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -5,14 +5,17 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import java.io.IOException; +import java.util.List; import java.util.Optional; class MockInvoker extends SearchInvoker { private final Coverage coverage; private Query query; + private List<Hit> hits; protected MockInvoker(int key, Coverage coverage) { super(Optional.of(new Node(key, "?", 0, 0))); @@ -23,6 +26,11 @@ class MockInvoker extends SearchInvoker { this(key, null); } + MockInvoker setHits(List<Hit> hits) { + this.hits = hits; + return this; + } + @Override protected void sendSearchRequest(Query query) throws IOException { this.query = query; @@ -34,6 +42,9 @@ class MockInvoker extends SearchInvoker { if (coverage != null) { ret.setCoverage(coverage); } + if (hits != null) { + ret.hits().addAll(hits); + } return ret; } |