diff options
Diffstat (limited to 'searchlib')
6 files changed, 171 insertions, 0 deletions
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index c55aadb5eae..5c6fd9ea6d0 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -232,6 +232,7 @@ vespa_define_module( install_java_artifact(searchlib) install_fat_java_artifact(searchlib) +install_java_artifact_dependencies(searchlib) vespa_install_script(src/main/sh/vespa-gbdt-converter bin) vespa_install_script(src/main/sh/vespa-treenet-converter bin) diff --git a/searchlib/pom.xml b/searchlib/pom.xml index 1d7862d541e..99ce2bdd0ba 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -57,6 +57,11 @@ </exclusions> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> <plugins> @@ -81,6 +86,10 @@ <artifactId>ph-javacc-maven-plugin</artifactId> </plugin> <plugin> + <groupId>com.github.os72</groupId> + <artifactId>protoc-jar-maven-plugin</artifactId> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <configuration> @@ -104,6 +113,22 @@ </executions> </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeArtifactIds>protobuf-java</includeArtifactIds> + </configuration> + </execution> + </executions> + </plugin> + <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>abi-check-plugin</artifactId> </plugin> diff --git a/searchlib/src/main/java/ai/vespa/searchlib/searchprotocol/protobuf/package-info.java b/searchlib/src/main/java/ai/vespa/searchlib/searchprotocol/protobuf/package-info.java new file mode 100644 index 00000000000..4463d4c9f52 --- /dev/null +++ b/searchlib/src/main/java/ai/vespa/searchlib/searchprotocol/protobuf/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.searchlib.searchprotocol.protobuf; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/protobuf/MapConverter.java b/searchlib/src/main/java/com/yahoo/searchlib/protobuf/MapConverter.java new file mode 100644 index 00000000000..4de8e3b5bb0 --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/protobuf/MapConverter.java @@ -0,0 +1,73 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.protobuf; + +import ai.vespa.searchlib.searchprotocol.protobuf.Search.StringProperty; +import ai.vespa.searchlib.searchprotocol.protobuf.Search.TensorProperty; +import com.google.protobuf.ByteString; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * @author ollivir + */ +public class MapConverter { + @FunctionalInterface + public interface PropertyInserter<T> { + void add(T prop); + } + + public static void convertMapTensors(Map<String, Object> map, PropertyInserter<TensorProperty.Builder> inserter) { + for (var entry : map.entrySet()) { + var value = entry.getValue(); + if (value instanceof Tensor) { + byte[] tensor = TypedBinaryFormat.encode((Tensor) value); + inserter.add(TensorProperty.newBuilder().setName(entry.getKey()).setValue(ByteString.copyFrom(tensor))); + } + } + } + + public static void convertMapStrings(Map<String, Object> map, PropertyInserter<StringProperty.Builder> inserter) { + for (var entry : map.entrySet()) { + var value = entry.getValue(); + if (!(value instanceof Tensor)) { + inserter.add(StringProperty.newBuilder().setName(entry.getKey()).addValues(value.toString())); + } + } + } + + public static void convertStringMultiMap(Map<String, List<String>> map, PropertyInserter<StringProperty.Builder> inserter) { + for (var entry : map.entrySet()) { + var values = entry.getValue(); + if (values != null) { + inserter.add(StringProperty.newBuilder().setName(entry.getKey()).addAllValues(values)); + } + } + } + + public static void convertMultiMap(Map<String, List<Object>> map, PropertyInserter<StringProperty.Builder> stringInserter, + PropertyInserter<TensorProperty.Builder> tensorInserter) { + for (var entry : map.entrySet()) { + if (entry.getValue() != null) { + var key = entry.getKey(); + var stringValues = new LinkedList<String>(); + for (var value : entry.getValue()) { + if (value != null) { + if (value instanceof Tensor) { + byte[] tensor = TypedBinaryFormat.encode((Tensor) value); + tensorInserter.add(TensorProperty.newBuilder().setName(key).setValue(ByteString.copyFrom(tensor))); + } else { + stringValues.add(value.toString()); + } + } + } + if (!stringValues.isEmpty()) { + stringInserter.add(StringProperty.newBuilder().setName(key).addAllValues(stringValues)); + } + } + } + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/protobuf/package-info.java b/searchlib/src/main/java/com/yahoo/searchlib/protobuf/package-info.java new file mode 100644 index 00000000000..bb7aeca35b9 --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/protobuf/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.searchlib.protobuf; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/searchlib/src/protobuf/search.proto b/searchlib/src/protobuf/search.proto new file mode 100644 index 00000000000..ea3daafe707 --- /dev/null +++ b/searchlib/src/protobuf/search.proto @@ -0,0 +1,62 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +syntax = "proto3"; + +package searchlib.searchprotocol.protobuf; + +option java_package = "ai.vespa.searchlib.searchprotocol.protobuf"; + +message Request { + int32 offset = 1; + int32 hits = 2; + int32 timeout = 3; // milliseconds + int32 trace_level = 4; + repeated SortField sorting = 5; + string session_key = 6; + string document_type = 7; + bool cache_grouping = 8; + bool cache_query = 9; + string rank_profile = 10; + repeated StringProperty feature_overrides = 11; + repeated TensorProperty tensor_feature_overrides = 12; + repeated StringProperty rank_properties = 13; + repeated TensorProperty tensor_rank_properties = 14; + repeated StringProperty highlight_terms = 15; + bytes grouping_blob = 16; // serialized opaquely like now, to be changed later + string geo_location = 17; // to be moved into query_tree + bytes query_tree_blob = 18; // serialized opaquely like now, to be changed later +} + +message TensorProperty { + string name = 1; + bytes value = 2; +} + +message StringProperty { + string name = 1; + repeated string values = 2; +} + +message SortField { + bool ascending = 1; + string field = 2; +} + +message Reply { + int64 total_hit_count = 1; + int64 coverage_docs = 2; + int64 active_docs = 3; + int64 soon_active_docs = 4; + bool degraded_by_match_phase = 5; + bool degraded_by_soft_timeout = 6; + + repeated Hit hits = 7; + + bytes grouping_blob = 8; // serialized opaquely like now, to be changed later +} + +message Hit { + bytes global_id = 1; + double relevance = 2; + bytes sort_data = 3; +} |