aboutsummaryrefslogtreecommitdiffstats
path: root/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
diff options
context:
space:
mode:
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java')
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java223
1 files changed, 0 insertions, 223 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
deleted file mode 100644
index 12be9a5c058..00000000000
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
+++ /dev/null
@@ -1,223 +0,0 @@
-package com.yahoo.search.dispatch.rpc;
-
-import ai.vespa.searchlib.searchprotocol.protobuf.Search;
-import ai.vespa.searchlib.searchprotocol.protobuf.Search.Request.Builder;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.yahoo.document.GlobalId;
-import com.yahoo.fs4.GetDocSumsPacket;
-import com.yahoo.io.GrowableByteBuffer;
-import com.yahoo.prelude.fastsearch.DocumentDatabase;
-import com.yahoo.prelude.fastsearch.FastHit;
-import com.yahoo.prelude.fastsearch.GroupingListHit;
-import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.grouping.vespa.GroupingExecutor;
-import com.yahoo.search.query.Model;
-import com.yahoo.search.query.Presentation;
-import com.yahoo.search.query.Ranking;
-import com.yahoo.search.query.Sorting;
-import com.yahoo.search.query.Sorting.Order;
-import com.yahoo.search.query.ranking.RankFeatures;
-import com.yahoo.search.query.ranking.RankProperties;
-import com.yahoo.search.result.Coverage;
-import com.yahoo.search.result.Relevance;
-import com.yahoo.searchlib.aggregation.Grouping;
-import com.yahoo.vespa.objects.BufferSerializer;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ProtobufSerialization {
- private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024;
-
- public static byte[] serializeQuery(Query query, String serverId, boolean includeQueryData) {
- return convertFromQuery(query, serverId, includeQueryData).toByteArray();
- }
-
- public static byte[] serializeResult(Result searchResult) {
- return convertFromResult(searchResult).toByteArray();
- }
-
- public static Result deserializeToResult(byte[] payload, Query query, VespaBackEndSearcher searcher)
- throws InvalidProtocolBufferException {
- var protobuf = Search.Reply.parseFrom(payload);
- var result = convertToResult(query, protobuf, searcher.getDocumentDatabase(query));
- return result;
- }
-
- private static Search.Request convertFromQuery(Query query, String serverId, boolean includeQueryData) {
- var builder = Search.Request.newBuilder().setHits(query.getHits()).setOffset(query.getOffset())
- .setTimeout((int) query.getTimeLeft());
-
- mergeToRequestFromRanking(query.getRanking(), builder, includeQueryData);
- mergeToRequestFromModel(query.getModel(), builder);
-
- if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) {
- // TODO verify that the session key is included whenever rank properties would have been
- builder.setSessionKey(query.getSessionId(serverId).toString());
- }
- if (query.properties().getBoolean(Model.ESTIMATE)) {
- builder.setHits(0);
- }
- if (GroupingExecutor.hasGroupingList(query)) {
- List<Grouping> groupingList = GroupingExecutor.getGroupingList(query);
- BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer());
- gbuf.putInt(null, groupingList.size());
- for (Grouping g : groupingList) {
- g.serialize(gbuf);
- }
- gbuf.getBuf().flip();
- builder.setGroupingBlob(ByteString.copyFrom(gbuf.getBuf().getByteBuffer()));
- }
-
- mergeToRequestFromPresentation(query.getPresentation(), builder, includeQueryData);
- if (query.getGroupingSessionCache()) {
- builder.setCacheGrouping(true);
- }
-
- return builder.build();
- }
-
- private static void mergeToRequestFromModel(Model model, Search.Request.Builder builder) {
- if (model.getDocumentDb() != null) {
- builder.setDocumentType(model.getDocumentDb());
- }
- int bufferSize = INITIAL_SERIALIZATION_BUFFER_SIZE;
- boolean success = false;
- while (!success) {
- try {
- ByteBuffer treeBuffer = ByteBuffer.allocate(bufferSize);
- model.getQueryTree().encode(treeBuffer);
- treeBuffer.flip();
- builder.setQueryTreeBlob(ByteString.copyFrom(treeBuffer));
- success = true;
- } catch (java.nio.BufferOverflowException e) {
- bufferSize *= 2;
- }
- }
- }
-
- private static void mergeToRequestFromPresentation(Presentation presentation, Search.Request.Builder builder,
- boolean includeQueryData) {
- if (includeQueryData && presentation.getHighlight() != null) {
- MapConverter.convertStringMultiMap(presentation.getHighlight().getHighlightTerms(), builder::addHighlightTerms);
- }
- }
-
- private static void mergeToRequestFromSorting(Sorting sorting, Search.Request.Builder builder, boolean includeQueryData) {
- for (var field : sorting.fieldOrders()) {
- var sortField = Search.SortField.newBuilder().setField(field.getSorter().getName())
- .setAscending(field.getSortOrder() == Order.ASCENDING).build();
- builder.addSorting(sortField);
- }
- }
-
- private static void mergeToRequestFromRanking(Ranking ranking, Search.Request.Builder builder, boolean includeQueryData) {
- builder.setRankProfile(ranking.getProfile());
- if (ranking.getQueryCache()) {
- builder.setCacheQuery(true);
- }
- if (ranking.getSorting() != null) {
- mergeToRequestFromSorting(ranking.getSorting(), builder, includeQueryData);
- }
- if (ranking.getLocation() != null) {
- builder.setGeoLocation(ranking.getLocation().toString());
- }
- mergeToRequestFromRankFeatures(ranking.getFeatures(), builder, includeQueryData);
- mergeToRequestFromRankProperties(ranking.getProperties(), builder, includeQueryData);
- }
-
- private static void mergeToRequestFromRankFeatures(RankFeatures features, Search.Request.Builder builder, boolean includeQueryData) {
- if (includeQueryData) {
- MapConverter.convertMapStrings(features.asMap(), builder::addFeatureOverrides);
- MapConverter.convertMapTensors(features.asMap(), builder::addTensorFeatureOverrides);
- }
- }
-
- private static void mergeToRequestFromRankProperties(RankProperties properties, Builder builder, boolean includeQueryData) {
- if (includeQueryData) {
- MapConverter.convertMultiMap(properties.asMap(), propB -> {
- if (!GetDocSumsPacket.sessionIdKey.equals(propB.getName())) {
- builder.addRankProperties(propB);
- }
- }, builder::addTensorRankProperties);
- }
- }
-
- private static Result convertToResult(Query query, Search.Reply protobuf, DocumentDatabase documentDatabase) {
- var result = new Result(query);
-
- result.setTotalHitCount(protobuf.getTotalHitCount());
- result.setCoverage(convertToCoverage(protobuf));
-
- if (protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty()) {
- ArrayList<Grouping> list = new ArrayList<>();
- BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer()));
- int cnt = buf.getInt(null);
- for (int i = 0; i < cnt; i++) {
- Grouping g = new Grouping();
- g.deserialize(buf);
- list.add(g);
- }
- GroupingListHit hit = new GroupingListHit(list, documentDatabase.getDocsumDefinitionSet());
- hit.setQuery(query);
- result.hits().add(hit);
- }
-
- for (var replyHit : protobuf.getHitsList()) {
- FastHit hit = new FastHit();
- hit.setQuery(query);
-
- hit.setRelevance(new Relevance(replyHit.getRelevance()));
- hit.setGlobalId(new GlobalId(replyHit.getGlobalId().toByteArray()));
-
- hit.setFillable();
- hit.setCached(false);
-
- result.hits().add(hit);
- }
-
- return result;
- }
-
- private static Coverage convertToCoverage(Search.Reply protobuf) {
- var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1);
- coverage.setNodesTried(1).setSoonActive(protobuf.getSoonActiveDocs());
-
- int degradedReason = 0;
- if (protobuf.getDegradedByMatchPhase())
- degradedReason |= Coverage.DEGRADED_BY_MATCH_PHASE;
- if (protobuf.getDegradedBySoftTimeout())
- degradedReason |= Coverage.DEGRADED_BY_TIMEOUT;
- coverage.setDegradedReason(degradedReason);
-
- return coverage;
- }
-
- private static Search.Reply convertFromResult(Result result) {
- var builder = Search.Reply.newBuilder();
-
- var coverage = result.getCoverage(false);
- if (coverage != null) {
- builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getSoonActive())
- .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()).setDegradedByMatchPhase(coverage.isDegradedByMatchPhase());
- }
-
- result.hits().iterator().forEachRemaining(hit -> {
- var hitBuilder = Search.Hit.newBuilder();
- if (hit.getRelevance() != null) {
- hitBuilder.setRelevance(hit.getRelevance().getScore());
- }
- if (hit instanceof FastHit) {
- FastHit fhit = (FastHit) hit;
- hitBuilder.setGlobalId(ByteString.copyFrom(fhit.getGlobalId().getRawId()));
- }
- builder.addHits(hitBuilder);
- });
- return builder.build();
- }
-
-}