diff options
Diffstat (limited to 'container-search/src/main/java/com')
18 files changed, 241 insertions, 229 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java b/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java index 023923d34ca..44691b04b84 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; -import com.yahoo.searchlib.protobuf.MapConverter; - import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; @@ -138,12 +135,6 @@ public class Highlight implements Cloneable { } } - public void addToProtobuf(Search.Request.Builder builder, boolean encodeQueryData) { - if(encodeQueryData) { - MapConverter.convertStringMultiMap(highlightTerms, builder::addHighlightTerms); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index b5871019bac..1fb199aa26a 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.protobuf.ByteString; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; -import com.yahoo.io.GrowableByteBuffer; import com.yahoo.log.LogLevel; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; @@ -17,7 +14,6 @@ import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.query.Model; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.query.Presentation; @@ -49,8 +45,6 @@ import com.yahoo.search.query.properties.RequestContextProperties; import com.yahoo.search.yql.NullItemException; import com.yahoo.search.yql.VespaSerializer; import com.yahoo.search.yql.YqlParser; -import com.yahoo.searchlib.aggregation.Grouping; -import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.yolean.Exceptions; import java.nio.ByteBuffer; @@ -1072,41 +1066,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return Collections.<String,Boolean>emptyMap(); } - public Search.Request toProtobuf(String serverId, boolean includeQueryData) { - var builder = Search.Request.newBuilder() - .setHits(hits) - .setOffset(offset) - .setTimeout((int) getTimeLeft()); - - ranking.addToProtobuf(builder, includeQueryData); - model.addToProtobuf(builder, includeQueryData); - - if(getGroupingSessionCache() || getRanking().getQueryCache()) { - // TODO verify that the session key is included whenever rank properties would have been - builder.setSessionKey(getSessionId(serverId).toString()); - } - if(properties().getBoolean(Model.ESTIMATE)) { - builder.setHits(0); - } - if(GroupingExecutor.hasGroupingList(this)) { - List<Grouping> groupingList = GroupingExecutor.getGroupingList(this); - BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer()); - gbuf.putInt(null, groupingList.size()); - for (Grouping g: groupingList){ - g.serialize(gbuf); - } - gbuf.getBuf().flip(); - builder.setGroupingBlob(ByteString.copyFrom(gbuf.getBuf().getByteBuffer())); - } - - presentation.addToProtobuf(builder, includeQueryData); - if(getGroupingSessionCache()) { - builder.setCacheGrouping(true); - } - - return builder.build(); - } - private Map<String, String> createModelMap() { Map<String, String> m = new HashMap<>(); if (model.getSearchPath() != null) m.put("searchpath", model.getSearchPath()); diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index e130ec6d6d5..4080b09f40b 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -1,15 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; -import com.google.protobuf.ByteString; import com.yahoo.collections.ListMap; -import com.yahoo.document.GlobalId; -import com.yahoo.io.GrowableByteBuffer; import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.DocumentDatabase; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.protect.Validator; import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.result.Coverage; @@ -18,12 +11,8 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.HitOrderer; import com.yahoo.search.result.HitSortOrderer; -import com.yahoo.search.result.Relevance; import com.yahoo.search.statistics.ElapsedTime; -import com.yahoo.searchlib.aggregation.Grouping; -import com.yahoo.vespa.objects.BufferSerializer; -import java.util.ArrayList; import java.util.Iterator; /** @@ -341,66 +330,4 @@ public final class Result extends com.yahoo.processing.Response implements Clone headers = new ListMap<>(); return headers; } - - public static Result fromProtobuf(Query query, Search.Reply protobuf, DocumentDatabase documentDatabase) { - var result = new Result(query); - - result.setTotalHitCount(protobuf.getTotalHitCount()); - result.setCoverage(Coverage.fromProtobuf(protobuf)); - - if(protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty()) { - ArrayList<Grouping> list = new ArrayList<>(); - BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); - int cnt = buf.getInt(null); - for (int i = 0; i < cnt; i++) { - Grouping g = new Grouping(); - g.deserialize(buf); - list.add(g); - } - GroupingListHit hit = new GroupingListHit(list, documentDatabase.getDocsumDefinitionSet()); - hit.setQuery(query); - result.hits().add(hit); - } - - for (var replyHit : protobuf.getHitsList()) { - FastHit hit = new FastHit(); - hit.setQuery(query); - - hit.setRelevance(new Relevance(replyHit.getRelevance())); - hit.setGlobalId(new GlobalId(replyHit.getGlobalId().toByteArray())); - - hit.setFillable(); - hit.setCached(false); - - result.hits().add(hit); - } - - return result; - } - - public Search.Reply toProtobuf() { - var builder = Search.Reply.newBuilder(); - - var coverage = getCoverage(false); - if(coverage != null) { - builder.setCoverageDocs(coverage.getDocs()) - .setActiveDocs(coverage.getActive()) - .setSoonActiveDocs(coverage.getSoonActive()) - .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()) - .setDegradedByMatchPhase(coverage.isDegradedByMatchPhase()); - } - - hits.iterator().forEachRemaining(hit -> { - var hitBuilder = Search.Hit.newBuilder(); - if (hit.getRelevance() != null) { - hitBuilder.setRelevance(hit.getRelevance().getScore()); - } - if (hit instanceof FastHit) { - FastHit fhit = (FastHit) hit; - hitBuilder.setGlobalId(ByteString.copyFrom(fhit.getGlobalId().getRawId())); - } - builder.addHits(hitBuilder); - }); - return builder.build(); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index a4c66671d43..0aee51e1e32 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -10,6 +10,8 @@ import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException; +import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index be625178d2f..019e07221a6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.CompressionType; import com.yahoo.prelude.fastsearch.FastHit; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java new file mode 100644 index 00000000000..7807b20f72d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -0,0 +1,223 @@ +package com.yahoo.search.dispatch.rpc; + +import ai.vespa.searchlib.searchprotocol.protobuf.Search; +import ai.vespa.searchlib.searchprotocol.protobuf.Search.Request.Builder; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.GetDocSumsPacket; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.prelude.fastsearch.DocumentDatabase; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.query.Model; +import com.yahoo.search.query.Presentation; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.ranking.RankFeatures; +import com.yahoo.search.query.ranking.RankProperties; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.Relevance; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.protobuf.MapConverter; +import com.yahoo.vespa.objects.BufferSerializer; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class ProtobufSerialization { + private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024; + + public static byte[] serializeQuery(Query query, String serverId, boolean includeQueryData) { + return convertFromQuery(query, serverId, includeQueryData).toByteArray(); + } + + private static Search.Request convertFromQuery(Query query, String serverId, boolean includeQueryData) { + var builder = Search.Request.newBuilder().setHits(query.getHits()).setOffset(query.getOffset()) + .setTimeout((int) query.getTimeLeft()); + + mergeToRequestFromRanking(query.getRanking(), builder, includeQueryData); + mergeToRequestFromModel(query.getModel(), builder, includeQueryData); + + if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { + // TODO verify that the session key is included whenever rank properties would have been + builder.setSessionKey(query.getSessionId(serverId).toString()); + } + if (query.properties().getBoolean(Model.ESTIMATE)) { + builder.setHits(0); + } + if (GroupingExecutor.hasGroupingList(query)) { + List<Grouping> groupingList = GroupingExecutor.getGroupingList(query); + BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer()); + gbuf.putInt(null, groupingList.size()); + for (Grouping g : groupingList) { + g.serialize(gbuf); + } + gbuf.getBuf().flip(); + builder.setGroupingBlob(ByteString.copyFrom(gbuf.getBuf().getByteBuffer())); + } + + mergeToRequestFromPresentation(query.getPresentation(), builder, includeQueryData); + if (query.getGroupingSessionCache()) { + builder.setCacheGrouping(true); + } + + return builder.build(); + } + + private static void mergeToRequestFromModel(Model model, Search.Request.Builder builder, boolean encodeQueryData) { + if (model.getDocumentDb() != null) { + builder.setDocumentType(model.getDocumentDb()); + } + int bufferSize = INITIAL_SERIALIZATION_BUFFER_SIZE; + boolean success = false; + while (!success) { + try { + ByteBuffer treeBuffer = ByteBuffer.allocate(bufferSize); + model.getQueryTree().encode(treeBuffer); + treeBuffer.flip(); + builder.setQueryTreeBlob(ByteString.copyFrom(treeBuffer)); + success = true; + } catch (java.nio.BufferOverflowException e) { + bufferSize *= 2; + } + } + } + + private static void mergeToRequestFromPresentation(Presentation presentation, Search.Request.Builder builder, boolean encodeQueryData) { + if (encodeQueryData && presentation.getHighlight() != null) { + MapConverter.convertStringMultiMap(presentation.getHighlight().getHighlightTerms(), builder::addHighlightTerms); + } + } + + private static void mergeToRequestFromSorting(Sorting sorting, Search.Request.Builder builder, boolean includeQueryData) { + for (var field : sorting.fieldOrders()) { + var sortField = Search.SortField.newBuilder().setField(field.getSorter().getName()) + .setAscending(field.getSortOrder() == Order.ASCENDING).build(); + builder.addSorting(sortField); + } + } + + private static void mergeToRequestFromRanking(Ranking ranking, Search.Request.Builder builder, boolean includeQueryData) { + builder.setRankProfile(ranking.getProfile()); + if (ranking.getQueryCache()) { + builder.setCacheQuery(true); + } + if (ranking.getSorting() != null) { + mergeToRequestFromSorting(ranking.getSorting(), builder, includeQueryData); + } + if (ranking.getLocation() != null) { + builder.setGeoLocation(ranking.getLocation().toString()); + } + mergeToRequestFromRankFeatures(ranking.getFeatures(), builder, includeQueryData); + mergeToRequestFromRankProperties(ranking.getProperties(), builder, includeQueryData); + } + + private static void mergeToRequestFromRankFeatures(RankFeatures features, Search.Request.Builder builder, boolean includeQueryData) { + if (includeQueryData) { + MapConverter.convertMapStrings(features.asMap(), builder::addFeatureOverrides); + MapConverter.convertMapTensors(features.asMap(), builder::addTensorFeatureOverrides); + } + } + + private static void mergeToRequestFromRankProperties(RankProperties properties, Builder builder, boolean includeQueryData) { + if (includeQueryData) { + MapConverter.convertMultiMap(properties.asMap(), propB -> { + if (!GetDocSumsPacket.sessionIdKey.equals(propB.getName())) { + builder.addRankProperties(propB); + } + }, builder::addTensorRankProperties); + } + } + + public static Result deserializeToResult(byte[] payload, Query query, VespaBackEndSearcher searcher) + throws InvalidProtocolBufferException { + var protobuf = Search.Reply.parseFrom(payload); + var result = convertToResult(query, protobuf, searcher.getDocumentDatabase(query)); + return result; + } + + private static Result convertToResult(Query query, Search.Reply protobuf, DocumentDatabase documentDatabase) { + var result = new Result(query); + + result.setTotalHitCount(protobuf.getTotalHitCount()); + result.setCoverage(convertToCoverage(protobuf)); + + if (protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty()) { + ArrayList<Grouping> list = new ArrayList<>(); + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); + int cnt = buf.getInt(null); + for (int i = 0; i < cnt; i++) { + Grouping g = new Grouping(); + g.deserialize(buf); + list.add(g); + } + GroupingListHit hit = new GroupingListHit(list, documentDatabase.getDocsumDefinitionSet()); + hit.setQuery(query); + result.hits().add(hit); + } + + for (var replyHit : protobuf.getHitsList()) { + FastHit hit = new FastHit(); + hit.setQuery(query); + + hit.setRelevance(new Relevance(replyHit.getRelevance())); + hit.setGlobalId(new GlobalId(replyHit.getGlobalId().toByteArray())); + + hit.setFillable(); + hit.setCached(false); + + result.hits().add(hit); + } + + return result; + } + + private static Coverage convertToCoverage(Search.Reply protobuf) { + var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1); + coverage.setNodesTried(1).setSoonActive(protobuf.getSoonActiveDocs()); + + int degradedReason = 0; + if (protobuf.getDegradedByMatchPhase()) + degradedReason |= Coverage.DEGRADED_BY_MATCH_PHASE; + if (protobuf.getDegradedBySoftTimeout()) + degradedReason |= Coverage.DEGRADED_BY_TIMEOUT; + coverage.setDegradedReason(degradedReason); + + return coverage; + } + + private static Search.Reply convertFromResult(Result result) { + var builder = Search.Reply.newBuilder(); + + var coverage = result.getCoverage(false); + if (coverage != null) { + builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getSoonActive()) + .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()).setDegradedByMatchPhase(coverage.isDegradedByMatchPhase()); + } + + result.hits().iterator().forEachRemaining(hit -> { + var hitBuilder = Search.Hit.newBuilder(); + if (hit.getRelevance() != null) { + hitBuilder.setRelevance(hit.getRelevance().getScore()); + } + if (hit instanceof FastHit) { + FastHit fhit = (FastHit) hit; + hitBuilder.setGlobalId(ByteString.copyFrom(fhit.getGlobalId().getRawId())); + } + builder.addHits(hitBuilder); + }); + return builder.build(); + } + + public static byte[] serializeResult(Result searchResult) { + return convertFromResult(searchResult).toByteArray(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index d6fe7ab631b..59564cddc61 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.CompressionType; import com.yahoo.jrt.DataValue; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 578c447dfbe..b7286997514 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; import com.yahoo.collections.ListMap; import com.yahoo.compress.CompressionType; @@ -12,6 +12,7 @@ import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.TimeoutException; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.query.SessionId; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 62b4ec2c57b..c8019278710 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,11 +1,15 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.FillInvoker; +import com.yahoo.search.dispatch.InterleavedSearchInvoker; +import com.yahoo.search.dispatch.InvokerFactory; +import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index e03e0875106..830ba45ef0f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -1,9 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.Compressor; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.Map; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 4ceac71ef63..88d77c760e3 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch; +package com.yahoo.search.dispatch.rpc; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.fs4.QueryPacket; @@ -9,7 +8,8 @@ import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.Client.SearchResponse; +import com.yahoo.search.dispatch.SearchInvoker; +import com.yahoo.search.dispatch.rpc.Client.SearchResponse; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; @@ -55,7 +55,7 @@ public class RpcSearchInvoker extends SearchInvoker { return; } - var payload = query.toProtobuf(searcher.getServerId(), true).toByteArray(); + var payload = ProtobufSerialization.serializeQuery(query, searcher.getServerId(), true); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(compression, payload); resourcePool.client().search(nodeConnection, compressionResult.type(), payload.length, compressionResult.data(), this, @@ -88,8 +88,7 @@ public class RpcSearchInvoker extends SearchInvoker { CompressionType compression = CompressionType.valueOf(searchResponse.compression()); byte[] payload = resourcePool.compressor().decompress(searchResponse.compressedPayload(), compression, searchResponse.uncompressedSize()); - var protobuf = Search.Reply.parseFrom(payload); - Result result = Result.fromProtobuf(query, protobuf, searcher.getDocumentDatabase(query)); + var result = ProtobufSerialization.deserializeToResult(payload, query, searcher); result.hits().unorderedIterator().forEachRemaining(hit -> { if(hit instanceof FastHit) { FastHit fhit = (FastHit) hit; diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 4e05b44b087..a874ed45e30 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; -import com.google.protobuf.ByteString; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.LocaleFactory; @@ -19,7 +17,6 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.searchchain.Execution; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -37,8 +34,6 @@ import static com.yahoo.text.Lowercase.toLowerCase; * @author bratseth */ public class Model implements Cloneable { - private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024; - /** The type representing the property arguments consumed by this */ private static final QueryProfileType argumentType; private static final CompoundName argumentTypeName; @@ -513,23 +508,4 @@ public class Model implements Cloneable { return false; } - public void addToProtobuf(Search.Request.Builder builder, boolean encodeQueryData) { - if (documentDbName != null) { - builder.setDocumentType(documentDbName); - } - int bufferSize = INITIAL_SERIALIZATION_BUFFER_SIZE; - boolean success = false; - while(!success) { - try { - ByteBuffer treeBuffer = ByteBuffer.allocate(bufferSize); - getQueryTree().encode(treeBuffer); - treeBuffer.flip(); - builder.setQueryTreeBlob(ByteString.copyFrom(treeBuffer)); - success = true; - } catch(java.nio.BufferOverflowException e) { - bufferSize *= 2; - } - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 5359569f5f3..6edef386d49 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.google.common.base.Splitter; import com.yahoo.collections.LazySet; import com.yahoo.component.ComponentSpecification; @@ -197,11 +196,5 @@ public class Presentation implements Cloneable { } - public void addToProtobuf(Search.Request.Builder builder, boolean encodeQueryData) { - if(highlight != null) { - highlight.addToProtobuf(builder, encodeQueryData); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 0d3aa01a85b..7444c94f491 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.yahoo.prelude.Freshness; import com.yahoo.prelude.Location; import com.yahoo.processing.request.CompoundName; @@ -269,19 +268,4 @@ public class Ranking implements Cloneable { } } - public void addToProtobuf(Search.Request.Builder builder, boolean includeQueryData) { - builder.setRankProfile(getProfile()); - if (queryCache) { - builder.setCacheQuery(true); - } - if(sorting != null) { - sorting.addToProtobuf(builder, includeQueryData); - } - if(location != null) { - builder.setGeoLocation(location.toString()); - } - rankFeatures.addToProtobuf(builder, includeQueryData); - rankProperties.addToProtobuf(builder, includeQueryData); - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index 3d575084214..b9d157de531 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.ibm.icu.text.Collator; import com.ibm.icu.util.ULocale; import com.yahoo.text.Utf8; @@ -405,14 +404,4 @@ public class Sorting implements Cloneable { return usedBytes; } - public void addToProtobuf(Search.Request.Builder builder, boolean includeQueryData) { - for (var field : fieldOrders) { - var sorting = Search.SortField.newBuilder() - .setField(field.getSorter().getName()) - .setAscending(field.getSortOrder() == Order.ASCENDING) - .build(); - builder.addSorting(sorting); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java index 9b0fd45c357..9786eba163a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.yahoo.fs4.MapEncoder; -import com.yahoo.searchlib.protobuf.MapConverter; import com.yahoo.tensor.Tensor; import com.yahoo.text.JSON; @@ -129,11 +127,4 @@ public class RankFeatures implements Cloneable { return JSON.encode(features); } - public void addToProtobuf(Search.Request.Builder builder, boolean includeQueryData) { - if (includeQueryData) { - MapConverter.convertMapStrings(features, builder::addFeatureOverrides); - MapConverter.convertMapTensors(features, builder::addTensorFeatureOverrides); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java index dc544386dc4..4158b0e7476 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java @@ -1,10 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.yahoo.fs4.GetDocSumsPacket; import com.yahoo.fs4.MapEncoder; -import com.yahoo.searchlib.protobuf.MapConverter; import com.yahoo.text.JSON; import java.nio.ByteBuffer; @@ -113,14 +111,4 @@ public class RankProperties implements Cloneable { return JSON.encode(properties); } - public void addToProtobuf(Search.Request.Builder builder, boolean includeQueryData) { - if (includeQueryData) { - MapConverter.convertMultiMap(properties, propB -> { - if (!GetDocSumsPacket.sessionIdKey.equals(propB.getName())) { - builder.addRankProperties(propB); - } - }, builder::addTensorRankProperties); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 30ea8f53fc0..25829b70b5e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.result; -import ai.vespa.searchlib.searchprotocol.protobuf.Search; import com.google.common.annotations.Beta; /** @@ -42,19 +41,4 @@ public class Coverage extends com.yahoo.container.handler.Coverage { public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; } public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } - - public static Coverage fromProtobuf(Search.Reply protobuf) { - var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1); - coverage.setNodesTried(1) - .setSoonActive(protobuf.getSoonActiveDocs()); - - int degradedReason = 0; - if (protobuf.getDegradedByMatchPhase()) - degradedReason |= Coverage.DEGRADED_BY_MATCH_PHASE; - if (protobuf.getDegradedBySoftTimeout()) - degradedReason |= Coverage.DEGRADED_BY_TIMEOUT; - coverage.setDegradedReason(degradedReason); - - return coverage; - } } |