diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
7 files changed, 92 insertions, 167 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 84e5e7d747f..388a31d1482 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -93,9 +93,10 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { - InvokerResult result = new InvokerResult(query, query.getHits()); - List<LeanHit> merged = Collections.emptyList(); + protected Result getSearchResult(Execution execution) throws IOException { + Result result = new Result(query); + List<Hit> merged = Collections.emptyList(); + List<Hit> auxiliary = new ArrayList<>(); long nextTimeout = query.getTimeLeft(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { @@ -104,7 +105,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); break; } else { - merged = mergeResult(result.getResult(), invoker.getSearchResult(execution), merged); + merged = mergeResult(result, invoker.getSearchResult(execution), merged, auxiliary); ejectInvoker(invoker); } nextTimeout = nextTimeout(); @@ -113,11 +114,12 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM throw new RuntimeException("Interrupted while waiting for search results", e); } - insertNetworkErrors(result.getResult()); - result.getResult().setCoverage(createCoverage()); + insertNetworkErrors(result); + result.setCoverage(createCoverage()); int needed = query.getOffset() + query.getHits(); + result.hits().addAll(auxiliary); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { - result.getLeanHits().add(merged.get(index)); + result.hits().add(merged.get(index)); } query.setOffset(0); // Now we are all trimmed down return result; @@ -189,31 +191,43 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current) { - collectCoverage(partialResult.getResult().getCoverage(true)); + private List<Hit> mergeResult(Result result, Result partialResult, List<Hit> current, List<Hit> auxiliaryHits) { + collectCoverage(partialResult.getCoverage(true)); - result.mergeWith(partialResult.getResult()); - List<Hit> partialNonLean = partialResult.getResult().hits().asUnorderedHits(); - for(Hit hit : partialNonLean) { - if (hit.isAuxiliary()) { - result.hits().add(hit); - } - } + result.mergeWith(partialResult); + List<Hit> partial = partialResult.hits().asUnorderedHits(); if (current.isEmpty() ) { - return partialResult.getLeanHits(); + boolean hasAuxillary = false; + for(Hit hit : partial) { + if (hit.isAuxiliary()) { + hasAuxillary = true; + break; + } + } + if ( ! hasAuxillary) + return partial; } - List<LeanHit> partial = partialResult.getLeanHits(); if (partial.isEmpty()) { return current; } int needed = query.getOffset() + query.getHits(); - List<LeanHit> merged = new ArrayList<>(needed); + List<Hit> merged = new ArrayList<>(needed); int indexCurrent = 0; int indexPartial = 0; while (indexCurrent < current.size() && indexPartial < partial.size() && merged.size() < needed) { - LeanHit incommingHit = partial.get(indexPartial); - LeanHit currentHit = current.get(indexCurrent); + Hit incommingHit = partial.get(indexPartial); + if (incommingHit.isAuxiliary()) { + auxiliaryHits.add(incommingHit); + indexPartial++; + continue; + } + Hit currentHit = current.get(indexCurrent); + if (currentHit.isAuxiliary()) { + auxiliaryHits.add(currentHit); + indexCurrent++; + continue; + } int cmpRes = currentHit.compareTo(incommingHit); if (cmpRes < 0) { @@ -228,11 +242,21 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM indexPartial++; } } - while ((indexCurrent < current.size()) && (merged.size() < needed)) { - merged.add(current.get(indexCurrent++)); + while (indexCurrent < current.size()) { + Hit h = current.get(indexCurrent++); + if (h.isAuxiliary()) { + auxiliaryHits.add(h); + } else if (merged.size() < needed) { + merged.add(h); + } } - while ((indexPartial < partial.size()) && (merged.size() < needed)) { - merged.add(partial.get(indexPartial++)); + while (indexPartial < partial.size()) { + Hit h = partial.get(indexPartial++); + if (h.isAuxiliary()) { + auxiliaryHits.add(h); + } else if (merged.size() < needed) { + merged.add(h); + } } return merged; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java deleted file mode 100644 index cd8228624c5..00000000000 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.yahoo.search.dispatch; - -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.query.Sorting; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Wraps a Result and a flat, skinny hit list - */ -public class InvokerResult { - private final Result result; - private final List<LeanHit> leanHits; - public InvokerResult(Result result) { - this.result = result; - this.leanHits = Collections.emptyList(); - } - public InvokerResult(Query query, int expectedHits) { - result = new Result(query); - leanHits = new ArrayList<>(expectedHits); - } - - public Result getResult() { - return result; - } - - public List<LeanHit> getLeanHits() { - return leanHits; - } - void complete() { - Query query = result.getQuery(); - Sorting sorting = query.getRanking().getSorting(); - for (LeanHit hit : leanHits) { - FastHit fh = new FastHit(hit.getGid(), hit.getRelevance(), hit.getPartId(), hit.getDistributionKey()); - if (hit.hasSortData()) { - fh.setSortData(hit.getSortData(), sorting); - } - if (hit.getQueryPacketData() != null) { - fh.setQueryPacketData(hit.getQueryPacketData()); - } - fh.setQuery(query); - fh.setFillable(); - fh.setCached(false); - result.hits().add(fh); - } - leanHits.clear(); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java deleted file mode 100644 index 3ce8459ac8f..00000000000 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.yahoo.search.dispatch; - -import com.yahoo.fs4.QueryPacketData; - -import java.util.Arrays; - -public class LeanHit implements Comparable<LeanHit> { - private final byte [] gid; - private final double relevance; - private final byte [] sortData; - private final int partId; - private final int distributionKey; - //TODO Remove when FS4 is gone - private QueryPacketData queryPacketData; - public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { - this.gid = gid; - this.relevance = relevance; - this.sortData = null; - this.partId = partId; - this.distributionKey = distributionKey; - } - public LeanHit(byte [] gid, int partId, int distributionKey, byte [] sortData) { - this.gid = gid; - this.relevance = Double.NEGATIVE_INFINITY; - this.sortData = sortData; - this.partId = partId; - this.distributionKey = distributionKey; - } - double getRelevance() { return relevance; } - byte [] getGid() { return gid; } - byte [] getSortData() { return sortData; } - boolean hasSortData() { return sortData != null; } - int getPartId() { return partId; } - int getDistributionKey() { return distributionKey; } - - QueryPacketData getQueryPacketData() { return queryPacketData; } - - public void setQueryPacketData(QueryPacketData queryPacketData) { - this.queryPacketData = queryPacketData; - } - - @Override - public int compareTo(LeanHit o) { - int res = (sortData != null) - ? compareData(sortData, o.sortData) - : Double.compare(o.relevance, relevance); - return (res != 0) ? res : compareData(gid, o.gid); - } - - private static int compareData(byte [] left, byte [] right) { - int i = Arrays.mismatch(left, right); - if (i < 0) { - return 0; - } - int max = Integer.min(left.length, right.length); - if (i >= max) { - return left.length - right.length; - } - int vl = (int) left[i] & 0xFF; - int vr = (int) right[i] & 0xFF; - return vl - vr; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 52a45dc421c..b1d25dddbb8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -42,12 +42,12 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected Result getSearchResult(Execution execution) throws IOException { Result res = new Result(query, message); if (coverage != null) { res.setCoverage(coverage); } - return new InvokerResult(res); + return res; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index 77b3df7c83a..3bfb9089457 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -32,15 +32,14 @@ public abstract class SearchInvoker extends CloseableInvoker { */ public Result search(Query query, Execution execution) throws IOException { sendSearchRequest(query); - InvokerResult result = getSearchResult(execution); - setFinalStatus(result.getResult().hits().getError() == null); - result.complete(); - return result.getResult(); + Result result = getSearchResult(execution); + setFinalStatus(result.hits().getError() == null); + return result; } protected abstract void sendSearchRequest(Query query) throws IOException; - protected abstract InvokerResult getSearchResult(Execution execution) throws IOException; + protected abstract Result getSearchResult(Execution execution) throws IOException; protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; @@ -56,12 +55,12 @@ public abstract class SearchInvoker extends CloseableInvoker { return node.map(Node::key); } - protected InvokerResult errorResult(Query query, ErrorMessage errorMessage) { + protected Result errorResult(Query query, ErrorMessage errorMessage) { Result error = new Result(query, errorMessage); Coverage errorCoverage = new Coverage(0, 0, 0); errorCoverage.setNodesTried(1); error.setCoverage(errorCoverage); - return new InvokerResult(error); + return error; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index cf59c3b7bec..6039e09bbe1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -15,8 +15,6 @@ import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.InvokerResult; -import com.yahoo.search.dispatch.LeanHit; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.query.Model; import com.yahoo.search.query.QueryTree; @@ -24,6 +22,7 @@ import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Sorting; import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.Relevance; import com.yahoo.searchlib.aggregation.Grouping; import com.yahoo.slime.BinaryFormat; import com.yahoo.vespa.objects.BufferSerializer; @@ -181,22 +180,31 @@ public class ProtobufSerialization { return convertFromResult(searchResult).toByteArray(); } - static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) + static Result deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) throws InvalidProtocolBufferException { var protobuf = SearchProtocol.SearchReply.parseFrom(payload); - return convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey); + return convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey, searcher.getName()); } - private static InvokerResult convertToResult(Query query, SearchProtocol.SearchReply protobuf, - DocumentDatabase documentDatabase, int partId, int distKey) - { - InvokerResult result = new InvokerResult(query, protobuf.getHitsCount()); + private static Result convertToResult(Query query, + SearchProtocol.SearchReply protobuf, + DocumentDatabase documentDatabase, + int partId, + int distKey, + String source) { + var result = new Result(query); - result.getResult().setTotalHitCount(protobuf.getTotalHitCount()); - result.getResult().setCoverage(convertToCoverage(protobuf)); + result.setTotalHitCount(protobuf.getTotalHitCount()); + result.setCoverage(convertToCoverage(protobuf)); + int hitItems = protobuf.getHitsCount(); var haveGrouping = protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { + hitItems++; + } + result.hits().ensureCapacity(hitItems); + + if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); int cnt = buf.getInt(null); ArrayList<Grouping> list = new ArrayList<>(cnt); @@ -207,16 +215,26 @@ public class ProtobufSerialization { } GroupingListHit hit = new GroupingListHit(list, documentDatabase.getDocsumDefinitionSet()); hit.setQuery(query); - result.getResult().hits().add(hit); + result.hits().add(hit); } + var sorting = query.getRanking().getSorting(); for (var replyHit : protobuf.getHitsList()) { - LeanHit hit = (!replyHit.getSortData().isEmpty()) - ? new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance()) - : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getSortData().toByteArray()); - result.getLeanHits().add(hit); - } + FastHit hit = new FastHit(replyHit.getGlobalId().toByteArray(), new Relevance(replyHit.getRelevance()), partId, distKey); + hit.setQuery(query); + if (!replyHit.getSortData().isEmpty()) { + hit.setSortData(replyHit.getSortData().toByteArray(), sorting); + } + hit.setFillable(); + hit.setCached(false); + hit.setSource(source); + + result.hits().add(hit); + } + if (sorting != null) { + result.hits().setSorted(true); + } var slimeTrace = protobuf.getSlimeTrace(); if (slimeTrace != null && !slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 59f17501c32..9f42e794f5e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -6,7 +6,6 @@ import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.searchcluster.Node; @@ -61,7 +60,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected Result getSearchResult(Execution execution) throws IOException { long timeLeftMs = query.getTimeLeft(); if (timeLeftMs <= 0) { return errorResult(query, ErrorMessage.createTimeout("Timeout while waiting for " + getName())); |