From e392280c624d6490028fe09e692c7f8da8831391 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 14:22:18 +0100 Subject: Add default constructor for TinyIdentitySet --- vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java index 1cc68777c88..bb540f82ea4 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java +++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java @@ -67,9 +67,12 @@ public final class TinyIdentitySet implements Set { * @param initSize * initial size of internal element array */ - public TinyIdentitySet(final int initSize) { + public TinyIdentitySet(int initSize) { entries = new Object[initSize]; } + public TinyIdentitySet() { + this(4); + } /** * Expose the index in the internal array of a given object. -1 is returned -- cgit v1.2.3 From a4901877fa39233edadb132c0e53079e49cb62b3 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 14:23:34 +0100 Subject: Allow for backend per schema. --- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 116 ++++++++++++--------- .../prelude/fastsearch/VespaBackEndSearcher.java | 3 +- .../prelude.fastsearch.documentdb-info.def | 22 +--- .../prelude/cluster/ClusterSearcherTestCase.java | 41 ++++---- 4 files changed, 88 insertions(+), 94 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1fe0835306e..fd69390a745 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; +import com.yahoo.collections.TinyIdentitySet; import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; @@ -31,6 +32,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -42,8 +44,6 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; import java.util.stream.Collectors; -import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; - /** * A searcher which forwards to a cluster of monitored native Vespa backends. * @@ -60,13 +60,12 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; // The set of document types contained in this search cluster - private final Set schemas; + private final Map schema2Searcher; private final SchemaInfo schemaInfo; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds - private final VespaBackEndSearcher server; private final Executor executor; private final GlobalPhaseRanker globalPhaseRanker; @@ -88,7 +87,7 @@ public class ClusterSearcher extends Searcher { searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = globalPhaseRanker; - schemas = new LinkedHashSet<>(); + schema2Searcher = new LinkedHashMap<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); @@ -97,17 +96,23 @@ public class ClusterSearcher extends Searcher { .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); - for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) - schemas.add(docDb.name()); - String uniqueServerId = UUID.randomUUID().toString(); - if (searchClusterConfig.indexingmode() == STREAMING) { - server = streamingCluster(uniqueServerId, searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); - vipStatus.addToRotation(server.getName()); - } else { - server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, - docSumParams, documentDbConfig, schemaInfo, dispatchers); + VespaBackEndSearcher streaming = null, indexed = null; + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { + if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { + if (indexed == null) { + indexed = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, + docSumParams, documentDbConfig, schemaInfo, dispatchers); + } + schema2Searcher.put(docDb.name(), indexed); + } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) { + if (streaming == null) { + streaming = streamingCluster(uniqueServerId, searchClusterIndex, + searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); + vipStatus.addToRotation(streaming.getName()); + } + schema2Searcher.put(docDb.name(), streaming); + } } } @@ -117,7 +122,8 @@ public class ClusterSearcher extends Searcher { return searchCluster; } } - return null; + throw new IllegalStateException("No configured search cluster '" + name + "' among : " + + config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList()); } private static ClusterParams makeClusterParams(int searchclusterIndex) { @@ -159,49 +165,57 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set schemas, VespaBackEndSearcher searcher, Executor executor) { + ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher, Executor executor) { this.schemaInfo = schemaInfo; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; - server = searcher; this.executor = executor; this.globalPhaseRanker = null; - this.schemas = schemas; + this.schema2Searcher = schema2Searcher; } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set schemas) { - this(schemaInfo, schemas, null, null); + ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher) { + this(schemaInfo, schema2Searcher, null); } @Override public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - var searcher = server; - if (searcher == null) { + if (schema2Searcher.isEmpty()) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } if (query.getTimeLeft() <= 0) { return new Result(query, ErrorMessage.createTimeout("No time left for searching")); } - return doSearch(searcher, query); + return doSearch(query); } @Override - public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { + public void fill(Result result, String summaryClass, Execution execution) { + fill(result, summaryClass); + } + private void fill(Result result, String summaryClass) { Query query = result.getQuery(); - - VespaBackEndSearcher searcher = server; - if (searcher != null) { - if (query.getTimeLeft() > 0) { - searcher.fill(result, summaryClass); - } else { - if (result.hits().getErrorHit() == null) { - result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + - query.getTimeout() + " ms")); + var restrict = query.getModel().getRestrict(); + Collection servers = (restrict != null && ! restrict.isEmpty()) + ? query.getModel().getRestrict().stream() + .map(schema2Searcher::get) + .collect(Collectors.toCollection(TinyIdentitySet::new)) + : schema2Searcher.values().stream().collect(Collectors.toCollection(TinyIdentitySet::new)); + + if ( ! servers.isEmpty() ) { + for (var server : servers) { + if (query.getTimeLeft() > 0) { + server.fill(result, summaryClass); + } else { + if (result.hits().getErrorHit() == null) { + result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + + query.getTimeout() + " ms")); + } } } } else { @@ -232,17 +246,17 @@ public class ClusterSearcher extends Searcher { query.getRanking().setQueryCache(false); } - private Result doSearch(VespaBackEndSearcher searcher, Query query) { - if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query); + private Result doSearch(Query query) { + if (schema2Searcher.size() > 1) { + return searchMultipleDocumentTypes(query); } else { - String schema = schemas.iterator().next(); + String schema = schema2Searcher.keySet().iterator().next(); query.getModel().setRestrict(schema); - return perSchemaSearch(searcher, schema, query); + return perSchemaSearch(schema, query); } } - private Result perSchemaSearch(VespaBackEndSearcher searcher, String schema, Query query) { + private Result perSchemaSearch(String schema, Query query) { Set restrict = query.getModel().getRestrict(); if (restrict.size() != 1) { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); @@ -258,7 +272,7 @@ public class ClusterSearcher extends Searcher { query.setOffset(0); query.setHits(useHits); } - Result result = searcher.search(schema, query); + Result result = schema2Searcher.get(schema).search(schema, query); if (useGlobalPhase) { globalPhaseRanker.rerankHits(query, result, schema); result.hits().trim(wantOffset, wantHits); @@ -285,17 +299,17 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(VespaBackEndSearcher searcher, Query query) { + private Result searchMultipleDocumentTypes(Query query) { Set schemas = resolveSchemas(query); Map schemaQueries = createQueries(query, schemas); if (schemaQueries.size() == 1) { var entry = schemaQueries.entrySet().iterator().next(); - return perSchemaSearch(searcher, entry.getKey(), entry.getValue()); + return perSchemaSearch(entry.getKey(), entry.getValue()); } else { Result mergedResult = new Result(query); List> pending = new ArrayList<>(schemaQueries.size()); for (var entry : schemaQueries.entrySet()) { - FutureTask task = new FutureTask<>(() -> perSchemaSearch(searcher, entry.getKey(), entry.getValue())); + FutureTask task = new FutureTask<>(() -> perSchemaSearch(entry.getKey(), entry.getValue())); try { executor.execute(task); pending.add(task); @@ -311,7 +325,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim @@ -328,7 +342,7 @@ public class ClusterSearcher extends Searcher { candidates.addAll(cluster.schemas()); } return (candidates.isEmpty() ? sources : candidates).stream() - .filter(schemas::contains).collect(Collectors.toUnmodifiableSet()); + .filter(schema2Searcher::containsKey).collect(Collectors.toUnmodifiableSet()); } Set resolveSchemas(Query query) { @@ -336,7 +350,7 @@ public class ClusterSearcher extends Searcher { if (restrict == null || restrict.isEmpty()) { Set sources = query.getModel().getSources(); return (sources == null || sources.isEmpty()) - ? schemas + ? schema2Searcher.keySet() : resolveSourceSubset(sources); } else { return filterValidDocumentTypes(restrict); @@ -346,7 +360,7 @@ public class ClusterSearcher extends Searcher { private Set filterValidDocumentTypes(Collection restrict) { Set retval = new LinkedHashSet<>(); for (String docType : restrict) { - if (docType != null && schemas.contains(docType)) { + if (docType != null && schema2Searcher.containsKey(docType)) { retval.add(docType); } } @@ -375,7 +389,11 @@ public class ClusterSearcher extends Searcher { @Override public void deconstruct() { - if (server != null) { + Map servers = new HashMap<>(); + for (var server : schema2Searcher.values()) { + servers.put(server.getName(), server); + } + for (var server : servers.values()) { server.shutDown(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 340262ed186..74f3c740524 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -184,7 +184,7 @@ public abstract class VespaBackEndSearcher { return result; } - private List partitionHits(Result result, String summaryClass) { + private static List partitionHits(Result result, String summaryClass) { List parts = new ArrayList<>(); TinyIdentitySet queryMap = new TinyIdentitySet<>(4); @@ -210,6 +210,7 @@ public abstract class VespaBackEndSearcher { return parts; } + //TODO Add schema here too. public void fill(Result result, String summaryClass) { if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove diff --git a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def index 3ef3e9ef824..71c88b79ecd 100644 --- a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def @@ -6,24 +6,4 @@ namespace=prelude.fastsearch ## The name of the schema/document database documentdb[].name string - -## The id of the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].id int -## The name of the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].name string -## The name of a field in the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].name string -## The type of a field in the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].type string -## Whether this field is a dynamic snippet. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].dynamic bool default=false - -## Information about rank profiles. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].name string -documentdb[].rankprofile[].hasSummaryFeatures bool default=true -documentdb[].rankprofile[].hasRankFeatures bool default=true - -# The name of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].input[].name string -# The tensor type of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].input[].type string +documentdb[].mode enum {INDEX, STREAMING, STORE_ONLY} default=INDEX diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index affa6161440..2b00bac5316 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -38,7 +39,6 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -51,21 +51,6 @@ public class ClusterSearcherTestCase { private static final double DELTA = 0.0000000000000001; - @Test - void testNoBackends() { - ClusterSearcher cluster = new ClusterSearcher(createSchemaInfo(), Set.of("dummy")); - try { - Execution execution = new Execution(cluster, Execution.Context.createContextStub()); - Query query = new Query("query=hello"); - query.setHits(10); - com.yahoo.search.Result result = execution.search(query); - assertNotNull(result.hits().getError()); - assertEquals("No backends in service. Try later", result.hits().getError().getMessage()); - } finally { - cluster.deconstruct(); - } - } - private static SchemaInfo createSchemaInfo() { var schemas = Stream.of("type1", "type2", "type3", "type4", "type5", "type6") .map(name -> new Schema.Builder(name).build()).toList(); @@ -89,10 +74,11 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolved() { + var backend = new MyMockSearcher(false); SchemaInfo schemaInfo = createSchemaInfo(); - ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { - ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Map.of("type6", backend)); try { assertEquals(Set.of("type1", "type2", "type3"), resolve(cluster1, "")); assertEquals(Set.of("type6"), resolve(type1, "")); @@ -129,10 +115,11 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolvedTODO_REMOVE() { + var backend = new MyMockSearcher(false); SchemaInfo schemaInfo = createSchemaInfo(); - ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { - ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Map.of("type6", backend)); try { assertEquals(Set.of(), resolve(cluster1, "&sources=cluster2")); } finally { @@ -267,10 +254,14 @@ public class ClusterSearcherTestCase { } private Execution createExecution(List docTypesList, boolean expectAttributePrefetch) { + var backend = new MyMockSearcher(expectAttributePrefetch); + Map searchers = new HashMap<>(); + for(String schema : docTypesList) { + searchers.put(schema, backend); + } Set documentTypes = new LinkedHashSet<>(docTypesList); ClusterSearcher cluster = new ClusterSearcher(toSchemaInfo(documentTypes, "mycluster"), - documentTypes, - new MyMockSearcher(expectAttributePrefetch), + searchers, new InThreadExecutorService()); try { List schemas = new ArrayList<>(); @@ -449,7 +440,11 @@ public class ClusterSearcherTestCase { clusterConfig.maxQueryCacheTimeout(maxQueryCacheTimeout); DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); - documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); + documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder() + .name("type1") + .mode(streamingMode + ? DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING + : DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX)); var schema = new Schema.Builder("type1"); DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); -- cgit v1.2.3 From ff0d5bc059ca79c194081082b44f2c6f52f8408f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 14:28:45 +0100 Subject: Searcher => Backend --- .../yahoo/schema/processing/ImplicitSummaries.java | 2 +- .../schema/processing/PredicateProcessor.java | 2 +- .../processing/ReferenceFieldsProcessor.java | 2 +- .../vespa/model/container/PlatformBundles.java | 6 +- .../configdefinitions/container.qr-searchers.def | 2 +- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 52 +-- .../com/yahoo/prelude/fastsearch/FastBackend.java | 160 +++++++++ .../com/yahoo/prelude/fastsearch/FastSearcher.java | 160 --------- .../prelude/fastsearch/VespaBackEndSearcher.java | 390 --------------------- .../com/yahoo/prelude/fastsearch/VespaBackend.java | 390 +++++++++++++++++++++ .../yahoo/prelude/searcher/BlendingSearcher.java | 4 +- .../java/com/yahoo/search/dispatch/Dispatcher.java | 10 +- .../com/yahoo/search/dispatch/InvokerFactory.java | 8 +- .../search/dispatch/rpc/ProtobufSerialization.java | 4 +- .../search/dispatch/rpc/RpcInvokerFactory.java | 6 +- .../search/dispatch/rpc/RpcSearchInvoker.java | 7 +- .../com/yahoo/search/searchchain/Execution.java | 4 +- .../java/com/yahoo/search/yql/FieldFiller.java | 2 +- .../vespa/streamingvisitors/MetricsSearcher.java | 2 +- .../vespa/streamingvisitors/StreamingBackend.java | 373 ++++++++++++++++++++ .../vespa/streamingvisitors/StreamingSearcher.java | 373 -------------------- .../vespa/streamingvisitors/StreamingVisitor.java | 2 +- .../prelude/cluster/ClusterSearcherTestCase.java | 14 +- .../fastsearch/test/FastSearcherTestCase.java | 22 +- .../fastsearch/test/PartialFillTestCase.java | 8 +- .../com/yahoo/search/dispatch/DispatcherTest.java | 12 +- .../search/dispatch/rpc/RpcSearchInvokerTest.java | 6 +- .../search/yql/YqlFieldAndSourceTestCase.java | 3 +- .../streamingvisitors/MetricsSearcherTestCase.java | 4 +- .../StreamingSearcherTestCase.java | 54 +-- 30 files changed, 1041 insertions(+), 1043 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java delete mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java delete mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java create mode 100644 container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java delete mode 100644 container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java index a05450aa3a0..816e5a074f6 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.logging.Level; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 7e9d79fc858..26107ad2dca 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java index e425f81c0b4..42457295895 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; /** * Class that processes reference fields and removes attribute aspect of such fields from summary. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 13e6ee6684d..f1c85699af5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -92,8 +92,8 @@ public class PlatformBundles { com.yahoo.docproc.SimpleDocumentProcessor.class.getName(), com.yahoo.language.simple.SimpleLinguistics.class.getName(), com.yahoo.prelude.cluster.ClusterSearcher.class.getName(), - com.yahoo.prelude.fastsearch.FastSearcher.class.getName(), - com.yahoo.prelude.fastsearch.VespaBackEndSearcher.class.getName(), + com.yahoo.prelude.fastsearch.FastBackend.class.getName(), + com.yahoo.prelude.fastsearch.VespaBackend.class.getName(), com.yahoo.prelude.querytransform.CJKSearcher.class.getName(), com.yahoo.prelude.querytransform.CollapsePhraseSearcher.class.getName(), com.yahoo.prelude.querytransform.LiteralBoostSearcher.class.getName(), @@ -144,7 +144,7 @@ public class PlatformBundles { com.yahoo.search.searchers.CacheControlSearcher.class.getName(), com.yahoo.search.searchers.RateLimitingSearcher.class.getName(), com.yahoo.vespa.streamingvisitors.MetricsSearcher.class.getName(), - com.yahoo.vespa.streamingvisitors.StreamingSearcher.class.getName() + com.yahoo.vespa.streamingvisitors.StreamingBackend.class.getName() ); } diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index f8871e9f315..2cd4af4fdf1 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -25,7 +25,7 @@ com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" ## Default docsum class the QR server should ask the backend to ## use for representing hints as default. -com.yahoo.prelude.fastsearch.FastSearcher.docsum.defaultclass string default="" +com.yahoo.prelude.fastsearch.FastBackend.docsum.defaultclass string default="" com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index fd69390a745..36e918c9ba3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -11,9 +11,9 @@ import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; +import com.yahoo.prelude.fastsearch.FastBackend; import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -25,7 +25,7 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.streamingvisitors.StreamingSearcher; +import com.yahoo.vespa.streamingvisitors.StreamingBackend; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; @@ -60,7 +60,7 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; // The set of document types contained in this search cluster - private final Map schema2Searcher; + private final Map schema2Searcher; private final SchemaInfo schemaInfo; private final long maxQueryTimeout; // in milliseconds @@ -97,7 +97,7 @@ public class ClusterSearcher extends Searcher { .defaultclass()); String uniqueServerId = UUID.randomUUID().toString(); - VespaBackEndSearcher streaming = null, indexed = null; + VespaBackend streaming = null, indexed = null; for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { if (indexed == null) { @@ -130,34 +130,34 @@ public class ClusterSearcher extends Searcher { return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); } - private static FastSearcher searchDispatch(int searchclusterIndex, - String searchClusterName, - String serverId, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - ComponentRegistry dispatchers) { + private static FastBackend searchDispatch(int searchclusterIndex, + String searchClusterName, + String serverId, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + ComponentRegistry dispatchers) { ClusterParams clusterParams = makeClusterParams(searchclusterIndex); ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName); Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId); if (dispatcher == null) throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured"); - return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + return new FastBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); } - private static StreamingSearcher streamingCluster(String serverId, - int searchclusterIndex, - QrSearchersConfig.Searchcluster searchClusterConfig, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - VespaDocumentAccess access) { + private static StreamingBackend streamingCluster(String serverId, + int searchclusterIndex, + QrSearchersConfig.Searchcluster searchClusterConfig, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + VespaDocumentAccess access) { if (searchClusterConfig.searchdef().size() != 1) throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " + searchClusterConfig.searchdef()); ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - StreamingSearcher searcher = new StreamingSearcher(access); + StreamingBackend searcher = new StreamingBackend(access); searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid()); searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); @@ -165,7 +165,7 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher, Executor executor) { + ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher, Executor executor) { this.schemaInfo = schemaInfo; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; @@ -176,7 +176,7 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher) { + ClusterSearcher(SchemaInfo schemaInfo, Map schema2Searcher) { this(schemaInfo, schema2Searcher, null); } @@ -201,7 +201,7 @@ public class ClusterSearcher extends Searcher { private void fill(Result result, String summaryClass) { Query query = result.getQuery(); var restrict = query.getModel().getRestrict(); - Collection servers = (restrict != null && ! restrict.isEmpty()) + Collection servers = (restrict != null && ! restrict.isEmpty()) ? query.getModel().getRestrict().stream() .map(schema2Searcher::get) .collect(Collectors.toCollection(TinyIdentitySet::new)) @@ -325,7 +325,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(mergedResult, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim @@ -389,7 +389,7 @@ public class ClusterSearcher extends Searcher { @Override public void deconstruct() { - Map servers = new HashMap<>(); + Map servers = new HashMap<>(); for (var server : schema2Searcher.values()) { servers.put(server.getName(), server); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java new file mode 100644 index 00000000000..b1b3483af8c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java @@ -0,0 +1,160 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.FillInvoker; +import com.yahoo.search.dispatch.SearchInvoker; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.schema.SchemaInfo; + +import java.io.IOException; +import java.util.Optional; + +/** + * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 + * network layer. + * + * @author bratseth + */ +// TODO: Clean up all the duplication in the various search methods by +// switching to doing all the error handling using exceptions below doSearch2. +// Right now half is done by exceptions handled in doSearch2 and half by setting +// errors on results and returning them. It could be handy to create a QueryHandlingErrorException +// or similar which could wrap an error message, and then just always throw that and +// catch and unwrap into a results with an error in high level methods. -Jon +public class FastBackend extends VespaBackend { + + /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ + private final Dispatcher dispatcher; + + /** + * Creates a Fastsearcher. + * + * @param serverId the resource pool used to create direct connections to the local search nodes when + * bypassing the dispatch node + * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. + * Eventually we will move everything to this protocol and never use dispatch nodes. + * At that point we won't need a cluster searcher above this to select and pass the right + * backend. + * @param docSumParams document summary parameters + * @param clusterParams the cluster number, and other cluster backend parameters + * @param documentdbInfoConfig document database parameters + */ + public FastBackend(String serverId, + Dispatcher dispatcher, + SummaryParameters docSumParams, + ClusterParams clusterParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo) { + init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + this.dispatcher = dispatcher; + } + + @Override + protected void transformQuery(Query query) { + QueryRewrite.rewriteSddocname(query); + } + + private void injectSource(HitGroup hits) { + for (Hit hit : hits.asUnorderedHits()) { + if (hit instanceof FastHit) { + hit.setSource(getName()); + } + } + } + + @Override + public Result doSearch2(String schema, Query query) { + if (dispatcher.allGroupsHaveSize1()) + forceSinglePassGrouping(query); + try (SearchInvoker invoker = getSearchInvoker(query)) { + Result result = invoker.search(query); + injectSource(result.hits()); + + if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { + // There is currently no correct choice for which + // summary class we want to fetch at this point. If we + // fetch the one selected by the user it may not + // contain the data we need. If we fetch the default + // one we end up fetching docsums twice unless the + // user also requested the default one. + fill(result, query.getPresentation().getSummary()); // ARGH + } + return result; + } catch (TimeoutException e) { + return new Result(query,ErrorMessage.createTimeout(e.getMessage())); + } catch (IOException e) { + Result result = new Result(query); + if (query.getTrace().getLevel() >= 1) + query.trace(getName() + " error response: " + result, false, 1); + result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); + return result; + } + } + + /** + * Perform a partial docsum fill for a temporary result + * representing a partition of the complete fill request. + * + * @param result result containing a partition of the unfilled hits + * @param summaryClass the summary class we want to fill with + **/ + @Override + protected void doPartialFill(Result result, String summaryClass) { + if (result.isFilled(summaryClass)) return; + + Query query = result.getQuery(); + traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); + + try (FillInvoker invoker = getFillInvoker(result)) { + invoker.fill(result, summaryClass); + } + } + + /** When we only search a single node, doing all grouping in one pass is more efficient */ + private void forceSinglePassGrouping(Query query) { + for (GroupingRequest groupingRequest : query.getSelect().getGrouping()) + forceSinglePassGrouping(groupingRequest.getRootOperation()); + } + + private void forceSinglePassGrouping(GroupingOperation operation) { + operation.setForceSinglePass(true); + for (GroupingOperation childOperation : operation.getChildren()) + forceSinglePassGrouping(childOperation); + } + + /** + * Returns an invocation object for use in a single search request. The specific implementation returned + * depends on query properties with the default being an invoker that interfaces with a dispatcher + * on the same host. + */ + private SearchInvoker getSearchInvoker(Query query) { + return dispatcher.getSearchInvoker(query, this); + } + + /** + * Returns an invocation object for use in a single fill request. The specific implementation returned + * depends on query properties with the default being an invoker that uses RPC to interface with + * content nodes. + */ + private FillInvoker getFillInvoker(Result result) { + return dispatcher.getFillInvoker(result, this); + } + + private static Optional quotedSummaryClass(String summaryClass) { + return Optional.of(summaryClass == null ? "[null]" : "'" + summaryClass + "'"); + } + + public String toString() { + return "fast searcher (" + getName() + ") "; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java deleted file mode 100644 index 85b7770a8b2..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.yahoo.prelude.querytransform.QueryRewrite; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.Dispatcher; -import com.yahoo.search.dispatch.FillInvoker; -import com.yahoo.search.dispatch.SearchInvoker; -import com.yahoo.search.grouping.GroupingRequest; -import com.yahoo.search.grouping.request.GroupingOperation; -import com.yahoo.search.query.Ranking; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.schema.SchemaInfo; - -import java.io.IOException; -import java.util.Optional; - -/** - * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 - * network layer. - * - * @author bratseth - */ -// TODO: Clean up all the duplication in the various search methods by -// switching to doing all the error handling using exceptions below doSearch2. -// Right now half is done by exceptions handled in doSearch2 and half by setting -// errors on results and returning them. It could be handy to create a QueryHandlingErrorException -// or similar which could wrap an error message, and then just always throw that and -// catch and unwrap into a results with an error in high level methods. -Jon -public class FastSearcher extends VespaBackEndSearcher { - - /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ - private final Dispatcher dispatcher; - - /** - * Creates a Fastsearcher. - * - * @param serverId the resource pool used to create direct connections to the local search nodes when - * bypassing the dispatch node - * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. - * Eventually we will move everything to this protocol and never use dispatch nodes. - * At that point we won't need a cluster searcher above this to select and pass the right - * backend. - * @param docSumParams document summary parameters - * @param clusterParams the cluster number, and other cluster backend parameters - * @param documentdbInfoConfig document database parameters - */ - public FastSearcher(String serverId, - Dispatcher dispatcher, - SummaryParameters docSumParams, - ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo) { - init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); - this.dispatcher = dispatcher; - } - - @Override - protected void transformQuery(Query query) { - QueryRewrite.rewriteSddocname(query); - } - - private void injectSource(HitGroup hits) { - for (Hit hit : hits.asUnorderedHits()) { - if (hit instanceof FastHit) { - hit.setSource(getName()); - } - } - } - - @Override - public Result doSearch2(String schema, Query query) { - if (dispatcher.allGroupsHaveSize1()) - forceSinglePassGrouping(query); - try (SearchInvoker invoker = getSearchInvoker(query)) { - Result result = invoker.search(query); - injectSource(result.hits()); - - if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { - // There is currently no correct choice for which - // summary class we want to fetch at this point. If we - // fetch the one selected by the user it may not - // contain the data we need. If we fetch the default - // one we end up fetching docsums twice unless the - // user also requested the default one. - fill(result, query.getPresentation().getSummary()); // ARGH - } - return result; - } catch (TimeoutException e) { - return new Result(query,ErrorMessage.createTimeout(e.getMessage())); - } catch (IOException e) { - Result result = new Result(query); - if (query.getTrace().getLevel() >= 1) - query.trace(getName() + " error response: " + result, false, 1); - result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); - return result; - } - } - - /** - * Perform a partial docsum fill for a temporary result - * representing a partition of the complete fill request. - * - * @param result result containing a partition of the unfilled hits - * @param summaryClass the summary class we want to fill with - **/ - @Override - protected void doPartialFill(Result result, String summaryClass) { - if (result.isFilled(summaryClass)) return; - - Query query = result.getQuery(); - traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); - - try (FillInvoker invoker = getFillInvoker(result)) { - invoker.fill(result, summaryClass); - } - } - - /** When we only search a single node, doing all grouping in one pass is more efficient */ - private void forceSinglePassGrouping(Query query) { - for (GroupingRequest groupingRequest : query.getSelect().getGrouping()) - forceSinglePassGrouping(groupingRequest.getRootOperation()); - } - - private void forceSinglePassGrouping(GroupingOperation operation) { - operation.setForceSinglePass(true); - for (GroupingOperation childOperation : operation.getChildren()) - forceSinglePassGrouping(childOperation); - } - - /** - * Returns an invocation object for use in a single search request. The specific implementation returned - * depends on query properties with the default being an invoker that interfaces with a dispatcher - * on the same host. - */ - private SearchInvoker getSearchInvoker(Query query) { - return dispatcher.getSearchInvoker(query, this); - } - - /** - * Returns an invocation object for use in a single fill request. The specific implementation returned - * depends on query properties with the default being an invoker that uses RPC to interface with - * content nodes. - */ - private FillInvoker getFillInvoker(Result result) { - return dispatcher.getFillInvoker(result, this); - } - - private static Optional quotedSummaryClass(String summaryClass) { - return Optional.of(summaryClass == null ? "[null]" : "'" + summaryClass + "'"); - } - - public String toString() { - return "fast searcher (" + getName() + ") "; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java deleted file mode 100644 index 74f3c740524..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.yahoo.collections.TinyIdentitySet; -import com.yahoo.fs4.DocsumPacket; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.GeoLocationItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; -import com.yahoo.prelude.querytransform.QueryRewrite; -import com.yahoo.protect.Validator; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.schema.RankProfile; -import com.yahoo.search.grouping.vespa.GroupingExecutor; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.searchlib.aggregation.Grouping; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * Superclass for backend searchers. - * - * @author baldersheim - */ -public abstract class VespaBackEndSearcher { - - /** for vespa-internal use only; consider renaming the summary class */ - public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; - - private String serverId; - - /** The set of all document databases available in the backend handled by this searcher */ - private final Map documentDbs = new LinkedHashMap<>(); - private DocumentDatabase defaultDocumentDb = null; - - /** Default docsum class. null means "unset" and is the default value */ - private String defaultDocsumClass = null; - - /** Returns an iterator which returns all hits below this result **/ - private static Iterator hitIterator(Result result) { - return result.hits().unorderedDeepIterator(); - } - - /** The name of this source */ - private String name; - - public final String getName() { return name; } - protected final String getDefaultDocsumClass() { return defaultDocsumClass; } - - /** Sets default document summary class. Default is null */ - private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } - - /** - * Searches a search cluster - * This is an endpoint - searchers will never propagate the search to any nested searcher. - * - * @param query the query to search - */ - protected abstract Result doSearch2(String schema, Query query); - - protected abstract void doPartialFill(Result result, String summaryClass); - - private boolean hasLocation(Item tree) { - if (tree instanceof GeoLocationItem) { - return true; - } - if (tree instanceof CompositeItem composite) { - for (Item child : composite.items()) { - if (hasLocation(child)) return true; - } - } - return false; - } - - /** - * Returns whether we need to send the query when fetching summaries. - * This is necessary if the query requests summary features or dynamic snippeting. - */ - public boolean summaryNeedsQuery(Query query) { - if (query.getRanking().getQueryCache()) return false; // Query is cached in backend - - DocumentDatabase documentDb = getDocumentDatabase(query); - - // Needed to generate a dynamic summary? - DocsumDefinition docsumDefinition = documentDb.getDocsumDefinitionSet().getDocsum(query.getPresentation().getSummary()); - if (docsumDefinition.isDynamic()) return true; - - if (hasLocation(query.getModel().getQueryTree())) return true; - - // Needed to generate ranking features? - RankProfile rankProfile = documentDb.schema().rankProfiles().get(query.getRanking().getProfile()); - if (rankProfile == null) return true; // stay safe - if (rankProfile.hasSummaryFeatures()) return true; - if (query.getRanking().getListFeatures()) return true; - - // (Don't just add other checks here as there is a return false above) - - return false; - } - - public String getServerId() { return serverId; } - - public DocumentDatabase getDocumentDatabase(Query query) { - if (query.getModel().getRestrict().size() == 1) { - String docTypeName = (String)query.getModel().getRestrict().toArray()[0]; - DocumentDatabase db = documentDbs.get(docTypeName); - if (db != null) { - return db; - } - } - return defaultDocumentDb; - } - - private void resolveDocumentDatabase(Query query) { - DocumentDatabase docDb = getDocumentDatabase(query); - if (docDb != null) { - query.getModel().setDocumentDb(docDb.schema().name()); - } - } - - public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { - this.serverId = serverId; - this.name = clusterParams.searcherName; - - Validator.ensureNotNull("Name of Vespa backend integration", getName()); - - setDefaultDocsumClass(docSumParams.defaultClass); - - if (documentdbInfoConfig != null) { - for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name())); - if (documentDbs.isEmpty()) - defaultDocumentDb = db; - documentDbs.put(docDb.name(), db); - } - } - } - - protected void transformQuery(Query query) { } - - public Result search(String schema, Query query) { - // query root should not be null here - Item root = query.getModel().getQueryTree().getRoot(); - if (root == null || root instanceof NullItem) { - return new Result(query, ErrorMessage.createNullQuery(query.getUri().toString())); - } - - if ( ! getDocumentDatabase(query).schema().rankProfiles().containsKey(query.getRanking().getProfile())) - return new Result(query, ErrorMessage.createInvalidQueryParameter(getDocumentDatabase(query).schema() + - " does not contain requested rank profile '" + - query.getRanking().getProfile() + "'")); - - QueryRewrite.optimizeByRestrict(query); - QueryRewrite.optimizeAndNot(query); - QueryRewrite.collapseSingleComposites(query); - - root = query.getModel().getQueryTree().getRoot(); - if (root == null || root instanceof NullItem) // root can become null after optimization - return new Result(query); - - resolveDocumentDatabase(query); - transformQuery(query); - traceQuery(name, "search", query, query.getOffset(), query.getHits(), 1, Optional.empty()); - - root = query.getModel().getQueryTree().getRoot(); - if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? - return new Result(query); - - Result result = doSearch2(schema, query); - - if (query.getTrace().getLevel() >= 1) - query.trace(getName() + " dispatch response: " + result, false, 1); - result.trace(getName()); - return result; - } - - private static List partitionHits(Result result, String summaryClass) { - List parts = new ArrayList<>(); - TinyIdentitySet queryMap = new TinyIdentitySet<>(4); - - for (Iterator i = hitIterator(result); i.hasNext(); ) { - Hit hit = i.next(); - if (hit instanceof FastHit fastHit) { - if ( ! fastHit.isFilled(summaryClass)) { - Query q = fastHit.getQuery(); - if (q == null) { - q = result.hits().getQuery(); // fallback for untagged hits - } - int idx = queryMap.indexOf(q); - if (idx < 0) { - idx = queryMap.size(); - Result r = new Result(q); - parts.add(r); - queryMap.add(q); - } - parts.get(idx).hits().add(fastHit); - } - } - } - return parts; - } - - //TODO Add schema here too. - public void fill(Result result, String summaryClass) { - if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove - - List parts = partitionHits(result, summaryClass); - if (!parts.isEmpty()) { // anything to fill at all? - for (Result r : parts) { - doPartialFill(r, summaryClass); - mergeErrorsInto(result, r); - } - result.hits().setSorted(false); - result.analyzeHits(); - } - } - - private void mergeErrorsInto(Result destination, Result source) { - destination.hits().addErrorsFrom(source.hits()); - } - - void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional quotedSummaryClass) { - if ((query.getTrace().getLevel() 0) { - return new FillHitResult(true, decodeSummary(summaryClass, hit, docsumdata)); - } - } - return new FillHitResult(false); - } - - static protected class FillHitsResult { - public final int skippedHits; // Number of hits not producing a summary. - public final String error; // Optional error message - FillHitsResult(int skippedHits, String error) { - this.skippedHits = skippedHits; - this.error = error; - } - } - /** - * Fills the hits. - * - * @return the number of hits that we did not return data for, and an optional error message. - * when things are working normally we return 0. - */ - protected FillHitsResult fillHits(Result result, DocsumPacket[] packets, String summaryClass) { - int skippedHits = 0; - String lastError = null; - int packetIndex = 0; - for (Iterator i = hitIterator(result); i.hasNext();) { - Hit hit = i.next(); - - if (hit instanceof FastHit fastHit && ! hit.isFilled(summaryClass)) { - DocsumPacket docsum = packets[packetIndex]; - - packetIndex++; - FillHitResult fr = fillHit(fastHit, docsum, summaryClass); - if ( ! fr.ok ) { - skippedHits++; - } - if (fr.error != null) { - result.hits().addError(ErrorMessage.createTimeout(fr.error)); - skippedHits++; - lastError = fr.error; - } - } - } - result.hits().setSorted(false); - return new FillHitsResult(skippedHits, lastError); - } - - private String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata) { - DocumentDatabase db = getDocumentDatabase(hit.getQuery()); - hit.setField(Hit.SDDOCNAME_FIELD, db.schema().name()); - return decodeSummary(summaryClass, hit, docsumdata, db.getDocsumDefinitionSet()); - } - - private static String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata, DocsumDefinitionSet docsumSet) { - String error = docsumSet.lazyDecode(summaryClass, docsumdata, hit); - if (error == null) { - hit.setFilled(summaryClass); - } - return error; - } - - public void shutDown() { } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java new file mode 100644 index 00000000000..5191bfc4f41 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -0,0 +1,390 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.collections.TinyIdentitySet; +import com.yahoo.fs4.DocsumPacket; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.GeoLocationItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.protect.Validator; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.searchlib.aggregation.Grouping; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Superclass for backend searchers. + * + * @author baldersheim + */ +public abstract class VespaBackend { + + /** for vespa-internal use only; consider renaming the summary class */ + public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; + + private String serverId; + + /** The set of all document databases available in the backend handled by this searcher */ + private final Map documentDbs = new LinkedHashMap<>(); + private DocumentDatabase defaultDocumentDb = null; + + /** Default docsum class. null means "unset" and is the default value */ + private String defaultDocsumClass = null; + + /** Returns an iterator which returns all hits below this result **/ + private static Iterator hitIterator(Result result) { + return result.hits().unorderedDeepIterator(); + } + + /** The name of this source */ + private String name; + + public final String getName() { return name; } + protected final String getDefaultDocsumClass() { return defaultDocsumClass; } + + /** Sets default document summary class. Default is null */ + private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } + + /** + * Searches a search cluster + * This is an endpoint - searchers will never propagate the search to any nested searcher. + * + * @param query the query to search + */ + protected abstract Result doSearch2(String schema, Query query); + + protected abstract void doPartialFill(Result result, String summaryClass); + + private boolean hasLocation(Item tree) { + if (tree instanceof GeoLocationItem) { + return true; + } + if (tree instanceof CompositeItem composite) { + for (Item child : composite.items()) { + if (hasLocation(child)) return true; + } + } + return false; + } + + /** + * Returns whether we need to send the query when fetching summaries. + * This is necessary if the query requests summary features or dynamic snippeting. + */ + public boolean summaryNeedsQuery(Query query) { + if (query.getRanking().getQueryCache()) return false; // Query is cached in backend + + DocumentDatabase documentDb = getDocumentDatabase(query); + + // Needed to generate a dynamic summary? + DocsumDefinition docsumDefinition = documentDb.getDocsumDefinitionSet().getDocsum(query.getPresentation().getSummary()); + if (docsumDefinition.isDynamic()) return true; + + if (hasLocation(query.getModel().getQueryTree())) return true; + + // Needed to generate ranking features? + RankProfile rankProfile = documentDb.schema().rankProfiles().get(query.getRanking().getProfile()); + if (rankProfile == null) return true; // stay safe + if (rankProfile.hasSummaryFeatures()) return true; + if (query.getRanking().getListFeatures()) return true; + + // (Don't just add other checks here as there is a return false above) + + return false; + } + + public String getServerId() { return serverId; } + + public DocumentDatabase getDocumentDatabase(Query query) { + if (query.getModel().getRestrict().size() == 1) { + String docTypeName = (String)query.getModel().getRestrict().toArray()[0]; + DocumentDatabase db = documentDbs.get(docTypeName); + if (db != null) { + return db; + } + } + return defaultDocumentDb; + } + + private void resolveDocumentDatabase(Query query) { + DocumentDatabase docDb = getDocumentDatabase(query); + if (docDb != null) { + query.getModel().setDocumentDb(docDb.schema().name()); + } + } + + public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams, + DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { + this.serverId = serverId; + this.name = clusterParams.searcherName; + + Validator.ensureNotNull("Name of Vespa backend integration", getName()); + + setDefaultDocsumClass(docSumParams.defaultClass); + + if (documentdbInfoConfig != null) { + for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { + DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name())); + if (documentDbs.isEmpty()) + defaultDocumentDb = db; + documentDbs.put(docDb.name(), db); + } + } + } + + protected void transformQuery(Query query) { } + + public Result search(String schema, Query query) { + // query root should not be null here + Item root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) { + return new Result(query, ErrorMessage.createNullQuery(query.getUri().toString())); + } + + if ( ! getDocumentDatabase(query).schema().rankProfiles().containsKey(query.getRanking().getProfile())) + return new Result(query, ErrorMessage.createInvalidQueryParameter(getDocumentDatabase(query).schema() + + " does not contain requested rank profile '" + + query.getRanking().getProfile() + "'")); + + QueryRewrite.optimizeByRestrict(query); + QueryRewrite.optimizeAndNot(query); + QueryRewrite.collapseSingleComposites(query); + + root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) // root can become null after optimization + return new Result(query); + + resolveDocumentDatabase(query); + transformQuery(query); + traceQuery(name, "search", query, query.getOffset(), query.getHits(), 1, Optional.empty()); + + root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? + return new Result(query); + + Result result = doSearch2(schema, query); + + if (query.getTrace().getLevel() >= 1) + query.trace(getName() + " dispatch response: " + result, false, 1); + result.trace(getName()); + return result; + } + + private static List partitionHits(Result result, String summaryClass) { + List parts = new ArrayList<>(); + TinyIdentitySet queryMap = new TinyIdentitySet<>(4); + + for (Iterator i = hitIterator(result); i.hasNext(); ) { + Hit hit = i.next(); + if (hit instanceof FastHit fastHit) { + if ( ! fastHit.isFilled(summaryClass)) { + Query q = fastHit.getQuery(); + if (q == null) { + q = result.hits().getQuery(); // fallback for untagged hits + } + int idx = queryMap.indexOf(q); + if (idx < 0) { + idx = queryMap.size(); + Result r = new Result(q); + parts.add(r); + queryMap.add(q); + } + parts.get(idx).hits().add(fastHit); + } + } + } + return parts; + } + + //TODO Add schema here too. + public void fill(Result result, String summaryClass) { + if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove + + List parts = partitionHits(result, summaryClass); + if (!parts.isEmpty()) { // anything to fill at all? + for (Result r : parts) { + doPartialFill(r, summaryClass); + mergeErrorsInto(result, r); + } + result.hits().setSorted(false); + result.analyzeHits(); + } + } + + private void mergeErrorsInto(Result destination, Result source) { + destination.hits().addErrorsFrom(source.hits()); + } + + void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional quotedSummaryClass) { + if ((query.getTrace().getLevel() 0) { + return new FillHitResult(true, decodeSummary(summaryClass, hit, docsumdata)); + } + } + return new FillHitResult(false); + } + + static protected class FillHitsResult { + public final int skippedHits; // Number of hits not producing a summary. + public final String error; // Optional error message + FillHitsResult(int skippedHits, String error) { + this.skippedHits = skippedHits; + this.error = error; + } + } + /** + * Fills the hits. + * + * @return the number of hits that we did not return data for, and an optional error message. + * when things are working normally we return 0. + */ + protected FillHitsResult fillHits(Result result, DocsumPacket[] packets, String summaryClass) { + int skippedHits = 0; + String lastError = null; + int packetIndex = 0; + for (Iterator i = hitIterator(result); i.hasNext();) { + Hit hit = i.next(); + + if (hit instanceof FastHit fastHit && ! hit.isFilled(summaryClass)) { + DocsumPacket docsum = packets[packetIndex]; + + packetIndex++; + FillHitResult fr = fillHit(fastHit, docsum, summaryClass); + if ( ! fr.ok ) { + skippedHits++; + } + if (fr.error != null) { + result.hits().addError(ErrorMessage.createTimeout(fr.error)); + skippedHits++; + lastError = fr.error; + } + } + } + result.hits().setSorted(false); + return new FillHitsResult(skippedHits, lastError); + } + + private String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata) { + DocumentDatabase db = getDocumentDatabase(hit.getQuery()); + hit.setField(Hit.SDDOCNAME_FIELD, db.schema().name()); + return decodeSummary(summaryClass, hit, docsumdata, db.getDocsumDefinitionSet()); + } + + private static String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata, DocsumDefinitionSet docsumSet) { + String error = docsumSet.lazyDecode(summaryClass, docsumdata, hit); + if (error == null) { + hit.setFilled(summaryClass); + } + return error; + } + + public void shutDown() { } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 1128ad9570d..634a16d1dee 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,7 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -115,7 +115,7 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { // TODO: remove or rename this internal summary class for Vespa 9 - execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + execution.fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 8ec88bd7f45..1689f6d246e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -6,7 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.annotation.Inject; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -256,13 +256,13 @@ public class Dispatcher extends AbstractComponent { } } - public FillInvoker getFillInvoker(Result result, VespaBackEndSearcher searcher) { + public FillInvoker getFillInvoker(Result result, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. return items.register(items.get().invokerFactory.createFillInvoker(searcher, result)); } } - public SearchInvoker getSearchInvoker(Query query, VespaBackEndSearcher searcher) { + public SearchInvoker getSearchInvoker(Query query, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. int maxHitsPerNode = dispatchConfig.maxHitsPerNode(); SearchInvoker invoker = getSearchPathInvoker(query, searcher, searchCluster.groupList(), items.get().invokerFactory, maxHitsPerNode) @@ -277,7 +277,7 @@ public class Dispatcher extends AbstractComponent { } /** Builds an invoker based on searchpath */ - private static Optional getSearchPathInvoker(Query query, VespaBackEndSearcher searcher, SearchGroups cluster, + private static Optional getSearchPathInvoker(Query query, VespaBackend searcher, SearchGroups cluster, InvokerFactory invokerFactory, int maxHitsPerNode) { String searchPath = query.getModel().getSearchPath(); if (searchPath == null) return Optional.empty(); @@ -297,7 +297,7 @@ public class Dispatcher extends AbstractComponent { } } - private static SearchInvoker getInternalInvoker(Query query, VespaBackEndSearcher searcher, SearchCluster cluster, + private static SearchInvoker getInternalInvoker(Query query, VespaBackend searcher, SearchCluster cluster, LoadBalancer loadBalancer, InvokerFactory invokerFactory, int maxHitsPerNode) { Optional directNode = cluster.localCorpusDispatchTarget(); if (directNode.isPresent()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f1d58122570..d7fad148c8c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -2,7 +2,7 @@ package com.yahoo.search.dispatch; import com.yahoo.concurrent.Timer; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; @@ -34,12 +34,12 @@ public abstract class InvokerFactory { this.hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); } - protected abstract Optional createNodeSearchInvoker(VespaBackEndSearcher searcher, + protected abstract Optional createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node); - public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); + public abstract FillInvoker createFillInvoker(VespaBackend searcher, Result result); /** * Creates a {@link SearchInvoker} for a list of content nodes. @@ -52,7 +52,7 @@ public abstract class InvokerFactory { * @return the invoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ - Optional createSearchInvoker(VespaBackEndSearcher searcher, + Optional createSearchInvoker(VespaBackend searcher, Query query, List nodes, boolean acceptIncompleteCoverage, diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 9da4c91cd16..284b4a4fcbc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -14,7 +14,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; @@ -227,7 +227,7 @@ public class ProtobufSerialization { return convertFromResult(searchResult).toByteArray(); } - static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) + static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackend searcher, int partId, int distKey) throws InvalidProtocolBufferException { var protobuf = SearchProtocol.SearchReply.parseFrom(payload); return convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 8221391afe6..9fd602032e9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.FillInvoker; @@ -37,12 +37,12 @@ public class RpcInvokerFactory extends InvokerFactory { } @Override - protected Optional createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, int maxHits, Node node) { + protected Optional createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node) { return Optional.of(new RpcSearchInvoker(searcher, compressor, node, rpcResourcePool, maxHits)); } @Override - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 93e09a04180..fb028358caf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -2,14 +2,13 @@ package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.Compressor; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.Optional; @@ -26,7 +25,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private static final String RPC_METHOD = "vespa.searchprotocol.search"; - private final VespaBackEndSearcher searcher; + private final VespaBackend searcher; private final Node node; private final RpcConnectionPool resourcePool; private final BlockingQueue> responses; @@ -35,7 +34,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private Query query; - RpcSearchInvoker(VespaBackEndSearcher searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { + RpcSearchInvoker(VespaBackend searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { super(Optional.of(node)); this.searcher = searcher; this.node = node; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0b3179f5ad6..6eb69c76afd 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -534,7 +534,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { */ @Deprecated // TODO Remove on Vespa 9. public void fillAttributes(Result result) { - fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 0685a77d7a9..64a989fd0e7 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; import com.yahoo.component.chain.dependencies.After; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index 5fef90c2012..e367bb58f80 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import static com.yahoo.vespa.streamingvisitors.StreamingSearcher.STREAMING_STATISTICS; +import static com.yahoo.vespa.streamingvisitors.StreamingBackend.STREAMING_STATISTICS; /** * Generates mail-specific query metrics. diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java new file mode 100644 index 00000000000..9953d76f50a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java @@ -0,0 +1,373 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.container.core.documentapi.VespaDocumentAccess; +import com.yahoo.document.DocumentId; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.select.parser.TokenMgrException; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.fs4.DocsumPacket; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.fastsearch.TimeoutException; +import com.yahoo.prelude.fastsearch.VespaBackend; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; +import com.yahoo.vespa.streamingvisitors.tracing.TraceDescription; + +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * The searcher which forwards queries to storage nodes using visiting. + * The searcher is a visitor client responsible for starting search + * visitors in storage and collecting and merging the results. + * + * @author baldersheim + * @author Ulf Carlin + */ +public class StreamingBackend extends VespaBackend { + + private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); + private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); + private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); + + static final String STREAMING_STATISTICS = "streaming.statistics"; + private final VisitorFactory visitorFactory; + private final TracingOptions tracingOptions; + private static final Logger log = Logger.getLogger(StreamingBackend.class.getName()); + + private Route route; + + /** The configId used to access the searchcluster. */ + private String searchClusterName = null; + + /** The route to the storage cluster. */ + private String storageClusterRouteSpec = null; + + StreamingBackend(VisitorFactory visitorFactory) { + this.visitorFactory = visitorFactory; + tracingOptions = TracingOptions.DEFAULT; + } + + StreamingBackend(VisitorFactory visitorFactory, TracingOptions tracingOptions) { + this.visitorFactory = visitorFactory; + this.tracingOptions = tracingOptions; + } + + public StreamingBackend(VespaDocumentAccess access) { + this(new VespaVisitorFactory(access)); + } + + private String getSearchClusterName() { return searchClusterName; } + private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } + public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; } + + public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { + this.storageClusterRouteSpec = storageClusterRouteSpec; + } + + @Override + protected void doPartialFill(Result result, String summaryClass) { + } + + private double durationInMillisFromNanoTime(long startTimeNanos) { + return (tracingOptions.getClock().nanoTimeNow() - startTimeNanos) / (double)TimeUnit.MILLISECONDS.toNanos(1); + } + + private boolean timeoutBadEnoughToBeReported(Query query, double durationMillis) { + return (durationMillis > (query.getTimeout() * tracingOptions.getTraceTimeoutMultiplierThreshold())); + } + + private static boolean queryIsLocationConstrained(Query query) { + return ((query.properties().getString(streamingUserid) != null) || + (query.properties().getString(streamingGroupname) != null)); + } + + private static int documentSelectionQueryParameterCount(Query query) { + int paramCount = 0; + if (query.properties().getString(streamingUserid) != null) + paramCount++; + if (query.properties().getString(streamingGroupname) != null) + paramCount++; + if (query.properties().getString(streamingSelection) != null) + paramCount++; + return paramCount; + } + + private boolean shouldTraceQuery(Query query) { + // Only trace for explicit bucket subset queries, as otherwise we'd get a trace entry for every superbucket in the system. + return (queryIsLocationConstrained(query) && + ((query.getTrace().getLevel() > 0) || tracingOptions.getSamplingStrategy().shouldSample())); + } + + private int inferEffectiveQueryTraceLevel(Query query) { + return ((query.getTrace().getLevel() == 0) && shouldTraceQuery(query)) // Honor query's explicit trace level if present. + ? tracingOptions.getTraceLevelOverride() + : query.getTrace().getLevel(); + } + + @Override + public Result doSearch2(String schema, Query query) { + if (query.getTimeLeft() <= 0) + return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); + + initializeMissingQueryFields(query); + if (documentSelectionQueryParameterCount(query) != 1) { + return new Result(query, ErrorMessage.createIllegalQuery("Streaming search requires either " + + "streaming.groupname or streaming.selection")); + } + + if (query.getTrace().isTraceable(4)) + query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + schema, 4); + long timeStartedNanos = tracingOptions.getClock().nanoTimeNow(); + int effectiveTraceLevel = inferEffectiveQueryTraceLevel(query); + + Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterName(), route, schema, effectiveTraceLevel); + try { + visitor.doSearch(); + } catch (ParseException e) { + return new Result(query, ErrorMessage.createInvalidQueryParameter("Failed to parse document selection string: " + + e.getMessage())); + } catch (TokenMgrException e) { + return new Result(query, ErrorMessage.createInvalidQueryParameter("Failed to tokenize document selection string: " + + e.getMessage())); + } catch (TimeoutException e) { + double elapsedMillis = durationInMillisFromNanoTime(timeStartedNanos); + if ((effectiveTraceLevel > 0) && timeoutBadEnoughToBeReported(query, elapsedMillis)) { + tracingOptions.getTraceExporter().maybeExport(() -> new TraceDescription(visitor.getTrace(), + String.format("Trace of %s which timed out after %.3g seconds", + query, elapsedMillis / 1000.0))); + } + return new Result(query, ErrorMessage.createTimeout(e.getMessage())); + } catch (InterruptedException e) { + return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage())); + } + return buildResultFromCompletedVisitor(query, visitor); + } + + private void initializeMissingQueryFields(Query query) { + lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec()); + + if (route == null) { + route = Route.parse(getStorageClusterRouteSpec()); + } + lazyTrace(query, 8, "Route is ", route); + + lazyTrace(query, 7, "doSearch2(): query docsum class=", + query.getPresentation().getSummary(), ", default docsum class=", + getDefaultDocsumClass()); + + if (query.getPresentation().getSummary() == null) { + lazyTrace(query, 6, + "doSearch2(): No summary class specified in query, using default: ", + getDefaultDocsumClass()); + query.getPresentation().setSummary(getDefaultDocsumClass()); + } else { + lazyTrace(query, 6, + "doSearch2(): Summary class has been specified in query: ", + query.getPresentation().getSummary()); + } + + lazyTrace(query, 8, "doSearch2(): rank properties=", query.getRanking()); + lazyTrace(query, 8, "doSearch2(): sort specification=", query + .getRanking().getSorting() == null ? null : query.getRanking() + .getSorting().fieldOrders()); + } + + private Result buildResultFromCompletedVisitor(Query query, Visitor visitor) { + lazyTrace(query, 8, "offset=", query.getOffset(), ", hits=", query.getHits()); + + Result result = new Result(query); + List hits = visitor.getHits(); // Sorted on rank + Map summaryMap = visitor.getSummaryMap(); + + lazyTrace(query, 7, "total hit count = ", visitor.getTotalHitCount(), + ", returned hit count = ", hits.size(), ", summary count = ", + summaryMap.size()); + + VisitorStatistics stats = visitor.getStatistics(); + result.setTotalHitCount(visitor.getTotalHitCount()); + result.setCoverage(new Coverage(stats.getDocumentsVisited(), stats.getDocumentsVisited(), 1, 1)); + query.trace(visitor.getStatistics().toString(), false, 2); + query.getContext(true).setProperty(STREAMING_STATISTICS, stats); + + DocsumPacket[] summaryPackets = new DocsumPacket [hits.size()]; + + int index = 0; + boolean skippedEarlierResult = false; + for (SearchResult.Hit hit : hits) { + if (!verifyDocId(hit.getDocId(), query, skippedEarlierResult)) { + skippedEarlierResult = true; + continue; + } + FastHit fastHit = buildSummaryHit(query, hit); + result.hits().add(fastHit); + + DocumentSummary.Summary summary = summaryMap.get(hit.getDocId()); + if (summary != null) { + DocsumPacket dp = new DocsumPacket(summary.getSummary()); + summaryPackets[index] = dp; + } else { + return new Result(query, ErrorMessage.createBackendCommunicationError("Did not find summary for hit with document id " + + hit.getDocId())); + } + + index++; + } + if (result.isFilled(query.getPresentation().getSummary())) { + lazyTrace(query, 8, "Result is filled for summary class ", query.getPresentation().getSummary()); + } else { + lazyTrace(query, 8, "Result is not filled for summary class ", query.getPresentation().getSummary()); + } + + List groupingList = visitor.getGroupings(); + lazyTrace(query, 8, "Grouping list=", groupingList); + if ( ! groupingList.isEmpty() ) { + GroupingListHit groupHit = new GroupingListHit(groupingList, getDocumentDatabase(query), query); + result.hits().add(groupHit); + } + + FillHitsResult fillHitsResult = fillHits(result, summaryPackets, query.getPresentation().getSummary()); + int skippedHits = fillHitsResult.skippedHits; + if (fillHitsResult.error != null) { + result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error)); + return result; + } + + if (skippedHits == 0) { + query.trace("All hits have been filled",4); // TODO: cache results or result.analyzeHits(); ? + } else { + lazyTrace(query, 8, "Skipping some hits for query: ", result.getQuery()); + } + + lazyTrace(query, 8, "Returning result ", result); + + if (skippedHits > 0) { + log.info("skipping " + skippedHits + " hits for query: " + result.getQuery()); + result.hits().addError(ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); + } + + return result; + } + + private FastHit buildSummaryHit(Query query, SearchResult.Hit hit) { + FastHit fastHit = new FastHit(); + fastHit.setQuery(query); + fastHit.setSource(getName()); + fastHit.setId(hit.getDocId()); + fastHit.setRelevance(new Relevance(hit.getRank())); + if (hit instanceof SearchResult.HitWithSortBlob sortedHit) { + fastHit.setSortData(sortedHit.getSortBlob(), query.getRanking().getSorting()); + } + if (hit.getMatchFeatures().isPresent()) { + fastHit.setField("matchfeatures", new FeatureData(hit.getMatchFeatures().get())); + } + + fastHit.setFillable(); + return fastHit; + } + + private static void lazyTrace(Query query, int level, Object... args) { + if (query.getTrace().isTraceable(level)) { + StringBuilder s = new StringBuilder(); + for (Object arg : args) { + s.append(arg); + } + query.trace(s.toString(), level); + } + } + + static boolean verifyDocId(String id, Query query, boolean skippedEarlierResult) { + String expectedUserId = query.properties().getString(streamingUserid); + String expectedGroupName = query.properties().getString(streamingGroupname); + + Level logLevel = Level.SEVERE; + if (skippedEarlierResult) { + logLevel = Level.FINE; + } + + DocumentId docId; + try { + docId = new DocumentId(id); + } catch (IllegalArgumentException iae) { + log.log(logLevel, "Bad result for " + query + ": " + iae.getMessage()); + return false; + } + + if (expectedUserId != null) { + long userId; + + if (docId.getScheme().hasNumber()) { + userId = docId.getScheme().getNumber(); + } else { + log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); + return false; + } + if (new BigInteger(expectedUserId).longValue() != userId) { + log.log(logLevel, "Got result with wrong user ID (expected " + expectedUserId + ") in document ID (" + + id + ") for " + query); + return false; + } + } else if (expectedGroupName != null) { + String groupName; + + if (docId.getScheme().hasGroup()) { + groupName = docId.getScheme().getGroup(); + } else { + log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); + return false; + } + if (!expectedGroupName.equals(groupName)) { + log.log(logLevel, "Got result with wrong group name (expected " + expectedGroupName + ") in document ID (" + + id + ") for " + query); + return false; + } + } + return true; + } + + public Pong ping(Ping ping, Execution execution) { + // TODO add a real pong + return new Pong(); + } + + private static class VespaVisitorFactory implements StreamingVisitor.VisitorSessionFactory, VisitorFactory { + + private final VespaDocumentAccess access; + + private VespaVisitorFactory(VespaDocumentAccess access) { + this.access = access; + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return access.createVisitorSession(params); + } + + @Override + public Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride) { + return new StreamingVisitor(query, searchCluster, route, schema, this, traceLevelOverride); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java deleted file mode 100644 index 2f3b48b5683..00000000000 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.streamingvisitors; - -import com.yahoo.container.core.documentapi.VespaDocumentAccess; -import com.yahoo.document.DocumentId; -import com.yahoo.document.select.parser.ParseException; -import com.yahoo.document.select.parser.TokenMgrException; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.fs4.DocsumPacket; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.fastsearch.TimeoutException; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Coverage; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.FeatureData; -import com.yahoo.search.result.Relevance; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.searchlib.aggregation.Grouping; -import com.yahoo.vdslib.DocumentSummary; -import com.yahoo.vdslib.SearchResult; -import com.yahoo.vdslib.VisitorStatistics; -import com.yahoo.vespa.streamingvisitors.tracing.TraceDescription; - -import java.math.BigInteger; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * The searcher which forwards queries to storage nodes using visiting. - * The searcher is a visitor client responsible for starting search - * visitors in storage and collecting and merging the results. - * - * @author baldersheim - * @author Ulf Carlin - */ -public class StreamingSearcher extends VespaBackEndSearcher { - - private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); - private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); - private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); - - static final String STREAMING_STATISTICS = "streaming.statistics"; - private final VisitorFactory visitorFactory; - private final TracingOptions tracingOptions; - private static final Logger log = Logger.getLogger(StreamingSearcher.class.getName()); - - private Route route; - - /** The configId used to access the searchcluster. */ - private String searchClusterName = null; - - /** The route to the storage cluster. */ - private String storageClusterRouteSpec = null; - - StreamingSearcher(VisitorFactory visitorFactory) { - this.visitorFactory = visitorFactory; - tracingOptions = TracingOptions.DEFAULT; - } - - StreamingSearcher(VisitorFactory visitorFactory, TracingOptions tracingOptions) { - this.visitorFactory = visitorFactory; - this.tracingOptions = tracingOptions; - } - - public StreamingSearcher(VespaDocumentAccess access) { - this(new VespaVisitorFactory(access)); - } - - private String getSearchClusterName() { return searchClusterName; } - private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } - public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; } - - public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { - this.storageClusterRouteSpec = storageClusterRouteSpec; - } - - @Override - protected void doPartialFill(Result result, String summaryClass) { - } - - private double durationInMillisFromNanoTime(long startTimeNanos) { - return (tracingOptions.getClock().nanoTimeNow() - startTimeNanos) / (double)TimeUnit.MILLISECONDS.toNanos(1); - } - - private boolean timeoutBadEnoughToBeReported(Query query, double durationMillis) { - return (durationMillis > (query.getTimeout() * tracingOptions.getTraceTimeoutMultiplierThreshold())); - } - - private static boolean queryIsLocationConstrained(Query query) { - return ((query.properties().getString(streamingUserid) != null) || - (query.properties().getString(streamingGroupname) != null)); - } - - private static int documentSelectionQueryParameterCount(Query query) { - int paramCount = 0; - if (query.properties().getString(streamingUserid) != null) - paramCount++; - if (query.properties().getString(streamingGroupname) != null) - paramCount++; - if (query.properties().getString(streamingSelection) != null) - paramCount++; - return paramCount; - } - - private boolean shouldTraceQuery(Query query) { - // Only trace for explicit bucket subset queries, as otherwise we'd get a trace entry for every superbucket in the system. - return (queryIsLocationConstrained(query) && - ((query.getTrace().getLevel() > 0) || tracingOptions.getSamplingStrategy().shouldSample())); - } - - private int inferEffectiveQueryTraceLevel(Query query) { - return ((query.getTrace().getLevel() == 0) && shouldTraceQuery(query)) // Honor query's explicit trace level if present. - ? tracingOptions.getTraceLevelOverride() - : query.getTrace().getLevel(); - } - - @Override - public Result doSearch2(String schema, Query query) { - if (query.getTimeLeft() <= 0) - return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); - - initializeMissingQueryFields(query); - if (documentSelectionQueryParameterCount(query) != 1) { - return new Result(query, ErrorMessage.createIllegalQuery("Streaming search requires either " + - "streaming.groupname or streaming.selection")); - } - - if (query.getTrace().isTraceable(4)) - query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + schema, 4); - long timeStartedNanos = tracingOptions.getClock().nanoTimeNow(); - int effectiveTraceLevel = inferEffectiveQueryTraceLevel(query); - - Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterName(), route, schema, effectiveTraceLevel); - try { - visitor.doSearch(); - } catch (ParseException e) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Failed to parse document selection string: " + - e.getMessage())); - } catch (TokenMgrException e) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Failed to tokenize document selection string: " + - e.getMessage())); - } catch (TimeoutException e) { - double elapsedMillis = durationInMillisFromNanoTime(timeStartedNanos); - if ((effectiveTraceLevel > 0) && timeoutBadEnoughToBeReported(query, elapsedMillis)) { - tracingOptions.getTraceExporter().maybeExport(() -> new TraceDescription(visitor.getTrace(), - String.format("Trace of %s which timed out after %.3g seconds", - query, elapsedMillis / 1000.0))); - } - return new Result(query, ErrorMessage.createTimeout(e.getMessage())); - } catch (InterruptedException e) { - return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage())); - } - return buildResultFromCompletedVisitor(query, visitor); - } - - private void initializeMissingQueryFields(Query query) { - lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec()); - - if (route == null) { - route = Route.parse(getStorageClusterRouteSpec()); - } - lazyTrace(query, 8, "Route is ", route); - - lazyTrace(query, 7, "doSearch2(): query docsum class=", - query.getPresentation().getSummary(), ", default docsum class=", - getDefaultDocsumClass()); - - if (query.getPresentation().getSummary() == null) { - lazyTrace(query, 6, - "doSearch2(): No summary class specified in query, using default: ", - getDefaultDocsumClass()); - query.getPresentation().setSummary(getDefaultDocsumClass()); - } else { - lazyTrace(query, 6, - "doSearch2(): Summary class has been specified in query: ", - query.getPresentation().getSummary()); - } - - lazyTrace(query, 8, "doSearch2(): rank properties=", query.getRanking()); - lazyTrace(query, 8, "doSearch2(): sort specification=", query - .getRanking().getSorting() == null ? null : query.getRanking() - .getSorting().fieldOrders()); - } - - private Result buildResultFromCompletedVisitor(Query query, Visitor visitor) { - lazyTrace(query, 8, "offset=", query.getOffset(), ", hits=", query.getHits()); - - Result result = new Result(query); - List hits = visitor.getHits(); // Sorted on rank - Map summaryMap = visitor.getSummaryMap(); - - lazyTrace(query, 7, "total hit count = ", visitor.getTotalHitCount(), - ", returned hit count = ", hits.size(), ", summary count = ", - summaryMap.size()); - - VisitorStatistics stats = visitor.getStatistics(); - result.setTotalHitCount(visitor.getTotalHitCount()); - result.setCoverage(new Coverage(stats.getDocumentsVisited(), stats.getDocumentsVisited(), 1, 1)); - query.trace(visitor.getStatistics().toString(), false, 2); - query.getContext(true).setProperty(STREAMING_STATISTICS, stats); - - DocsumPacket[] summaryPackets = new DocsumPacket [hits.size()]; - - int index = 0; - boolean skippedEarlierResult = false; - for (SearchResult.Hit hit : hits) { - if (!verifyDocId(hit.getDocId(), query, skippedEarlierResult)) { - skippedEarlierResult = true; - continue; - } - FastHit fastHit = buildSummaryHit(query, hit); - result.hits().add(fastHit); - - DocumentSummary.Summary summary = summaryMap.get(hit.getDocId()); - if (summary != null) { - DocsumPacket dp = new DocsumPacket(summary.getSummary()); - summaryPackets[index] = dp; - } else { - return new Result(query, ErrorMessage.createBackendCommunicationError("Did not find summary for hit with document id " + - hit.getDocId())); - } - - index++; - } - if (result.isFilled(query.getPresentation().getSummary())) { - lazyTrace(query, 8, "Result is filled for summary class ", query.getPresentation().getSummary()); - } else { - lazyTrace(query, 8, "Result is not filled for summary class ", query.getPresentation().getSummary()); - } - - List groupingList = visitor.getGroupings(); - lazyTrace(query, 8, "Grouping list=", groupingList); - if ( ! groupingList.isEmpty() ) { - GroupingListHit groupHit = new GroupingListHit(groupingList, getDocumentDatabase(query), query); - result.hits().add(groupHit); - } - - FillHitsResult fillHitsResult = fillHits(result, summaryPackets, query.getPresentation().getSummary()); - int skippedHits = fillHitsResult.skippedHits; - if (fillHitsResult.error != null) { - result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error)); - return result; - } - - if (skippedHits == 0) { - query.trace("All hits have been filled",4); // TODO: cache results or result.analyzeHits(); ? - } else { - lazyTrace(query, 8, "Skipping some hits for query: ", result.getQuery()); - } - - lazyTrace(query, 8, "Returning result ", result); - - if (skippedHits > 0) { - log.info("skipping " + skippedHits + " hits for query: " + result.getQuery()); - result.hits().addError(ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); - } - - return result; - } - - private FastHit buildSummaryHit(Query query, SearchResult.Hit hit) { - FastHit fastHit = new FastHit(); - fastHit.setQuery(query); - fastHit.setSource(getName()); - fastHit.setId(hit.getDocId()); - fastHit.setRelevance(new Relevance(hit.getRank())); - if (hit instanceof SearchResult.HitWithSortBlob sortedHit) { - fastHit.setSortData(sortedHit.getSortBlob(), query.getRanking().getSorting()); - } - if (hit.getMatchFeatures().isPresent()) { - fastHit.setField("matchfeatures", new FeatureData(hit.getMatchFeatures().get())); - } - - fastHit.setFillable(); - return fastHit; - } - - private static void lazyTrace(Query query, int level, Object... args) { - if (query.getTrace().isTraceable(level)) { - StringBuilder s = new StringBuilder(); - for (Object arg : args) { - s.append(arg); - } - query.trace(s.toString(), level); - } - } - - static boolean verifyDocId(String id, Query query, boolean skippedEarlierResult) { - String expectedUserId = query.properties().getString(streamingUserid); - String expectedGroupName = query.properties().getString(streamingGroupname); - - Level logLevel = Level.SEVERE; - if (skippedEarlierResult) { - logLevel = Level.FINE; - } - - DocumentId docId; - try { - docId = new DocumentId(id); - } catch (IllegalArgumentException iae) { - log.log(logLevel, "Bad result for " + query + ": " + iae.getMessage()); - return false; - } - - if (expectedUserId != null) { - long userId; - - if (docId.getScheme().hasNumber()) { - userId = docId.getScheme().getNumber(); - } else { - log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); - return false; - } - if (new BigInteger(expectedUserId).longValue() != userId) { - log.log(logLevel, "Got result with wrong user ID (expected " + expectedUserId + ") in document ID (" + - id + ") for " + query); - return false; - } - } else if (expectedGroupName != null) { - String groupName; - - if (docId.getScheme().hasGroup()) { - groupName = docId.getScheme().getGroup(); - } else { - log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); - return false; - } - if (!expectedGroupName.equals(groupName)) { - log.log(logLevel, "Got result with wrong group name (expected " + expectedGroupName + ") in document ID (" + - id + ") for " + query); - return false; - } - } - return true; - } - - public Pong ping(Ping ping, Execution execution) { - // TODO add a real pong - return new Pong(); - } - - private static class VespaVisitorFactory implements StreamingVisitor.VisitorSessionFactory, VisitorFactory { - - private final VespaDocumentAccess access; - - private VespaVisitorFactory(VespaDocumentAccess access) { - this.access = access; - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { - return access.createVisitorSession(params); - } - - @Override - public Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride) { - return new StreamingVisitor(query, searchCluster, route, schema, this, traceLevelOverride); - } - - } - -} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java index e638e0cf12a..bb155e3f934 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java @@ -267,7 +267,7 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { log.log(Level.FINE, () -> "StreamingVisitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); - throw new TimeoutException("Query timed out in " + StreamingSearcher.class.getName()); + throw new TimeoutException("Query timed out in " + StreamingBackend.class.getName()); } } finally { session.destroy(); diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 2b00bac5316..d52fbe580a7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -9,7 +9,7 @@ import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; @@ -74,7 +74,7 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolved() { - var backend = new MyMockSearcher(false); + var backend = new MyMockBackend(false); SchemaInfo schemaInfo = createSchemaInfo(); ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { @@ -115,7 +115,7 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolvedTODO_REMOVE() { - var backend = new MyMockSearcher(false); + var backend = new MyMockBackend(false); SchemaInfo schemaInfo = createSchemaInfo(); ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { @@ -130,7 +130,7 @@ public class ClusterSearcherTestCase { } } - private static class MyMockSearcher extends VespaBackEndSearcher { + private static class MyMockBackend extends VespaBackend { private final String type1 = "type1"; private final String type2 = "type2"; @@ -187,7 +187,7 @@ public class ClusterSearcherTestCase { createHit(getId(type3, 2), 5))); } - MyMockSearcher(boolean expectAttributePrefetch) { + MyMockBackend(boolean expectAttributePrefetch) { this.expectAttributePrefetch = expectAttributePrefetch; init(); } @@ -254,8 +254,8 @@ public class ClusterSearcherTestCase { } private Execution createExecution(List docTypesList, boolean expectAttributePrefetch) { - var backend = new MyMockSearcher(expectAttributePrefetch); - Map searchers = new HashMap<>(); + var backend = new MyMockBackend(expectAttributePrefetch); + Map searchers = new HashMap<>(); for(String schema : docTypesList) { searchers.put(schema, backend); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 8270700a66b..3928743c954 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -6,9 +6,9 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; +import com.yahoo.prelude.fastsearch.FastBackend; import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.MockDispatcher; @@ -44,8 +44,8 @@ public class FastSearcherTestCase { @Test void testNullQuery() { - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - FastSearcher fastSearcher = new FastSearcher("container.0", + Logger.getLogger(FastBackend.class.getName()).setLevel(Level.ALL); + FastBackend fastSearcher = new FastBackend("container.0", MockDispatcher.create(List.of()), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -62,7 +62,7 @@ public class FastSearcherTestCase { assertEquals(Error.NULL_QUERY.code, message.getCode()); } - private Result doSearch(VespaBackEndSearcher searcher, Query query, int offset, int hits) { + private Result doSearch(VespaBackend searcher, Query query, int offset, int hits) { query.setOffset(offset); query.setHits(hits); return searcher.search(SCHEMA, query); @@ -70,7 +70,7 @@ public class FastSearcherTestCase { @Test void testSinglePassGroupingIsForcedWithSingleNodeGroups() { - FastSearcher fastSearcher = new FastSearcher("container.0", + FastBackend fastSearcher = new FastBackend("container.0", MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -93,7 +93,7 @@ public class FastSearcherTestCase { @Test void testRankProfileValidation() { - FastSearcher fastSearcher = new FastSearcher("container.0", + FastBackend fastSearcher = new FastBackend("container.0", MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -112,7 +112,7 @@ public class FastSearcherTestCase { .add(new RankProfile.Builder("default").setHasRankFeatures(false) .setHasSummaryFeatures(false) .build()); - FastSearcher backend = new FastSearcher("container.0", + FastBackend backend = new FastBackend("container.0", MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -132,7 +132,7 @@ public class FastSearcherTestCase { void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0), new Node(CLUSTER, 2, "host1", 0))); - FastSearcher fastSearcher = new FastSearcher("container.0", + FastBackend fastSearcher = new FastBackend("container.0", dispatcher, new SummaryParameters(null), new ClusterParams("testhittype"), @@ -182,11 +182,11 @@ public class FastSearcherTestCase { assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus } - private String searchError(String query, VespaBackEndSearcher searcher) { + private String searchError(String query, VespaBackend searcher) { return search(query, searcher).hits().getError().getDetailedMessage(); } - private Result search(String query, VespaBackEndSearcher searcher) { + private Result search(String query, VespaBackend searcher) { return searcher.search(SCHEMA, new Query(query)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java index a033ccdace6..cf5a0ca7bd9 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.component.chain.Chain; import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.*; */ public class PartialFillTestCase { - public static class FS4 extends VespaBackEndSearcher { + public static class FS4 extends VespaBackend { public List history = new ArrayList<>(); protected Result doSearch2(String schema, Query query) { return new Result(query); @@ -32,7 +32,7 @@ public class PartialFillTestCase { } } - public static class BadFS4 extends VespaBackEndSearcher { + public static class BadFS4 extends VespaBackend { protected Result doSearch2(String schema, Query query) { return new Result(query); } @@ -137,7 +137,7 @@ public class PartialFillTestCase { } } - private void doFill(VespaBackEndSearcher searcher, Result result, String summaryClass) { + private void doFill(VespaBackend searcher, Result result, String summaryClass) { searcher.fill(result, summaryClass); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 23846db3f0b..c966fbc200d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -3,7 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.compress.CompressionType; import com.yahoo.prelude.Pong; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; @@ -236,7 +236,7 @@ public class DispatcherTest { // This factory just forwards search to the dummy RPC layer above, nothing more. InvokerFactoryFactory invokerFactories = (rpcConnectionPool, searchGroups, dispatchConfig) -> new InvokerFactory(searchGroups, dispatchConfig) { - @Override protected Optional createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, int maxHits, Node node) { + @Override protected Optional createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node) { return Optional.of(new SearchInvoker(Optional.of(node)) { @Override protected Object sendSearchRequest(Query query, Object context) { rpcPool.getConnection(node.key()).request(null, null, 0, null, null, 0); @@ -248,7 +248,7 @@ public class DispatcherTest { @Override protected void release() { } }); }; - @Override public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + @Override public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { return new FillInvoker() { @Override protected void getFillResults(Result result, String summaryClass) { fail(); } @Override protected void sendFillRequest(Result result, String summaryClass) { fail(); } @@ -369,7 +369,7 @@ public class DispatcherTest { } @Override - public Optional createSearchInvoker(VespaBackEndSearcher searcher, + public Optional createSearchInvoker(VespaBackend searcher, Query query, List nodes, boolean acceptIncompleteCoverage, @@ -391,7 +391,7 @@ public class DispatcherTest { } @Override - protected Optional createNodeSearchInvoker(VespaBackEndSearcher searcher, + protected Optional createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHitsPerNode, Node node) { @@ -400,7 +400,7 @@ public class DispatcherTest { } @Override - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { fail("Unexpected call to createFillInvoker"); return null; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index 2600d8612f8..ef8e0522337 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -5,7 +5,7 @@ package com.yahoo.search.dispatch.rpc; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.CompressionType; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Node; @@ -117,8 +117,8 @@ public class RpcSearchInvokerTest { }; } - private VespaBackEndSearcher mockSearcher() { - return new VespaBackEndSearcher() { + private VespaBackend mockSearcher() { + return new VespaBackend() { @Override protected Result doSearch2(String schema, Query query) { fail("Unexpected call"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 4455c8a04a5..87d18c18db5 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.*; import java.util.Arrays; import java.util.List; -import java.util.Map; import com.yahoo.search.schema.DocumentSummary; import com.yahoo.search.schema.Schema; @@ -21,7 +20,7 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; /** diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java index 831261bb91a..5cc0e6b060b 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java @@ -134,9 +134,9 @@ public class MetricsSearcherTestCase { private void assignContextProperties(Query query, String loadType) { if (loadType != null && loadType.equals(LOADTYPE1)) { - query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, visitorStats); + query.getContext(true).setProperty(StreamingBackend.STREAMING_STATISTICS, visitorStats); } else { - query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, null); + query.getContext(true).setProperty(StreamingBackend.STREAMING_STATISTICS, null); } } } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index c7e95f802ab..b96dd97f76c 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -159,7 +159,7 @@ public class StreamingSearcherTestCase { } } - private static Result executeQuery(StreamingSearcher searcher, Query query) { + private static Result executeQuery(StreamingBackend searcher, Query query) { return searcher.doSearch2("test", query); } @@ -181,7 +181,7 @@ public class StreamingSearcherTestCase { return queries; } - private static void checkError(StreamingSearcher searcher, String queryString, String message, String detailedMessage) { + private static void checkError(StreamingBackend searcher, String queryString, String message, String detailedMessage) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNotNull(result.hits().getError()); @@ -194,7 +194,7 @@ public class StreamingSearcherTestCase { } } - private static void checkSearch(StreamingSearcher searcher, String queryString, int hitCount, String idPrefix) { + private static void checkSearch(StreamingBackend searcher, String queryString, int hitCount, String idPrefix) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNull(result.hits().getError()); @@ -212,11 +212,11 @@ public class StreamingSearcherTestCase { } } - private static void checkGrouping(StreamingSearcher searcher, String queryString, int hitCount) { + private static void checkGrouping(StreamingBackend searcher, String queryString, int hitCount) { checkSearch(searcher, queryString, hitCount, null); } - private static void checkMatchFeatures(StreamingSearcher searcher) { + private static void checkMatchFeatures(StreamingBackend searcher) { String queryString = "/?streaming.selection=true&query=match_features"; Result result = executeQuery(searcher, new Query(queryString)); assertNull(result.hits().getError()); @@ -229,7 +229,7 @@ public class StreamingSearcherTestCase { @Test void testBasics() { MockVisitorFactory factory = new MockVisitorFactory(); - StreamingSearcher searcher = new StreamingSearcher(factory); + StreamingBackend searcher = new StreamingBackend(factory); var schema = new Schema.Builder("test"); schema.add(new com.yahoo.search.schema.DocumentSummary.Builder("default").build()); @@ -276,25 +276,25 @@ public class StreamingSearcherTestCase { String groupId2 = "id:namespace:mytype:g=group2:userspecific"; String badId = "unknowscheme:namespace:something"; - assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, true)); - - assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(userId2, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId1, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId2, generalQuery, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, generalQuery, false)); - - assertTrue(StreamingSearcher.verifyDocId(userId1, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(userId2, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId1, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId2, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, user1Query, false)); - - assertFalse(StreamingSearcher.verifyDocId(userId1, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(userId2, group1Query, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId1, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId2, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, group1Query, false)); + assertTrue(StreamingBackend.verifyDocId(userId1, generalQuery, true)); + + assertTrue(StreamingBackend.verifyDocId(userId1, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(userId2, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(groupId1, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(groupId2, generalQuery, false)); + assertFalse(StreamingBackend.verifyDocId(badId, generalQuery, false)); + + assertTrue(StreamingBackend.verifyDocId(userId1, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(userId2, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId1, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId2, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(badId, user1Query, false)); + + assertFalse(StreamingBackend.verifyDocId(userId1, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(userId2, group1Query, false)); + assertTrue(StreamingBackend.verifyDocId(groupId1, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId2, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(badId, group1Query, false)); } private static class TraceFixture { @@ -304,13 +304,13 @@ public class StreamingSearcherTestCase { TracingOptions options; MockVisitorFactory factory; - StreamingSearcher searcher; + StreamingBackend searcher; private TraceFixture(Long firstTimestamp, Long... additionalTimestamps) { clock = MockUtils.mockedClockReturning(firstTimestamp, additionalTimestamps); options = new TracingOptions(sampler, exporter, clock, 8, 2.0); factory = new MockVisitorFactory(); - searcher = new StreamingSearcher(factory, options); + searcher = new StreamingBackend(factory, options); } private TraceFixture() { -- cgit v1.2.3 From 0f7a4e4b0ae2902535249cb7a9d9e1acbf6dd503 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 15:05:07 +0100 Subject: Correctly produce index mode per schema too. --- .../yahoo/schema/derived/DerivedConfiguration.java | 6 +++++ .../yahoo/vespa/model/search/SearchCluster.java | 7 ++++-- .../search/test/DocumentDatabaseTestCase.java | 27 ++++++++++++++++++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index e303e878550..b35918b3460 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -28,6 +28,7 @@ import java.io.Writer; public class DerivedConfiguration { private final Schema schema; + private final boolean isStreaming; private Summaries summaries; private Juniperrc juniperrc; private AttributeFields attributeFields; @@ -67,6 +68,7 @@ public class DerivedConfiguration { * schema is later modified. */ public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { + this.isStreaming = isStreaming; try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -150,6 +152,10 @@ public class DerivedConfiguration { } } + public boolean isStreaming() { + return isStreaming; + } + public Summaries getSummaries() { return summaries; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index c0aec7ba0b4..f19af8b62ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -113,8 +113,11 @@ public abstract class SearchCluster extends TreeConfigProducer sds = List.of(DocType.create("a", "index"), DocType.create("b", "streaming")); + var tester = new SchemaTester(); + var model = tester.createModel(sds, ""); + DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); + assertEquals(1, indexed_cfg.documentdb().size()); + var db = indexed_cfg.documentdb(0); + assertEquals("a", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); + DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b"); + assertEquals(1, streaming_cfg.documentdb().size()); + db = streaming_cfg.documentdb(0); + assertEquals("b", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); + } + private void assertAttributesConfigIndependentOfMode(String mode, List sds, List documentDBConfigIds, Map> expectedAttributesMap) -- cgit v1.2.3 From d51144a9aa9be319acb1a0b3677a403b8f5fc358 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 15:43:18 +0100 Subject: Correct naming --- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 2 +- .../yahoo/prelude/test/IntegrationTestCase.java | 174 --------------------- 2 files changed, 1 insertion(+), 175 deletions(-) delete mode 100644 container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 36e918c9ba3..28ba4fb09b2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -93,7 +93,7 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); SummaryParameters docSumParams = new SummaryParameters(qrsConfig - .com().yahoo().prelude().fastsearch().FastSearcher().docsum() + .com().yahoo().prelude().fastsearch().FastBackend().docsum() .defaultclass()); String uniqueServerId = UUID.randomUUID().toString(); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java deleted file mode 100644 index 0a3e7bda318..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.test; - -import com.yahoo.search.result.Hit; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.jupiter.api.Test; - -/** - * Runs a query thru the configured search chain from a real http channel - * to a mock fdispatch channel. The setup is rather complicated, as the point is - * to span as much of the processing from query to result as possible. - * - * @author bratseth - */ -public class IntegrationTestCase { - - public static class SecondSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result = execution.search(query); - result.hits().add(new Hit("searcher:2",1000)); - return result; - } - } - public static class ThirdSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result = execution.search(query); - result.hits().add(new Hit("searcher:3",1000)); - return result; - } - } - - @Test - void testQuery() throws java.io.IOException { - /* - TODO: (JSB) This blocks forever on Linux (not Windows) because - the ServerSocketChannel.accept method in Server - seems to starve the test running thread, - causing it to get stuck in waitForServerInitialization. - This must be caused by starvation because - replacing the test with Thread.sleep(n) - gives the same result if n is large enough (2000 - is large enough, 1000 is not. - Resolve this in some way, perhaps by switching to - non-blocking io (and then remember to remove the next line). - */ - } - - /* - if (1==1) return; - ServerThread serverThread=new ServerThread(); - try { - serverThread.start(); - waitForServerInitialization(); - insertMockFs4Channel(); - ByteBuffer buffer=ByteBuffer.allocate(4096); - buffer.put(getBytes("GET /?query=hans HTTP/1.1\n\n")); - SocketChannel socket= - SocketChannel.open(new InetSocketAddress(Server.get().getHost(), - Server.get().getPort())); - buffer.flip(); - socket.write(buffer); - - buffer.clear(); - socket.read(buffer); - // TODO: Validate return too - - } - finally { - serverThread.interrupt(); - } - } - - private static void assertCorrectQueryData(QueryPacket packet) { - assertEquals("Query x packet " + - "[query: query 'RANK hans bcatpat.bidxpatlvl1:hans' [path='/']]", - packet.toString()); - } - - private void insertMockFs4Channel() { - Searcher current=SearchChain.get(); - while (current.getChained().getChained()!=null) - current=current.getChained(); - assertTrue(current.getChained() instanceof FastSearcher); - FastSearcher mockFastSearcher= - new FastSearcher(new MockFSChannel(), - "file:etc/qr-summary.cf", - "testhittype"); - current.setChained(mockFastSearcher); - } - - private void waitForServerInitialization() { - int sleptMs=0; - while (Server.get().getHost()==null) { - try { Thread.sleep(10); } catch (Exception e) {} - sleptMs+=10; - } - } - - private class ServerThread extends Thread { - - public void run() { - try { - Server.get().start(8081,new SearchRequestHandler()); - } - catch (java.io.IOException e) { - throw new RuntimeException("Failed",e); - } - } - } - - private byte[] getBytes(String string) { - try { - return string.getBytes("utf-8"); - } - catch (java.io.UnsupportedEncodingException e) { - throw new RuntimeException("Won't happen",e); - } - } - */ - /** A channel which returns hardcoded packets of the same type as fdispatch */ - /* - private static class MockFSChannel extends Channel { - - public MockFSChannel() {} - - public void sendPacket(Packet packet) { - if (packet instanceof QueryPacket) { - assertCorrectQueryData((QueryPacket)packet); - } - else { - throw new RuntimeException("Mock channel don't know what to reply to " + - packet); - } - } - - public Packet[] receivePackets() { - List packets=new java.util.ArrayList(); - QueryResultPacket result=QueryResultPacket.create(); - result.addDocument(new DocumentInfo(123,2003,234,1000,1)); - result.addDocument(new DocumentInfo(456,1855,234,1001,1)); - packets.add(result); - addDocsums(packets); - return (Packet[])packets.toArray(new Packet[packets.size()]); - } - - private void addDocsums(List packets) { - ByteBuffer buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); - buffer.position(0); - packets.add(PacketDecoder.decode(buffer)); - - buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); - buffer.position(0); - packets.add(PacketDecoder.decode(buffer)); - - packets.add(EolPacket.create()); - } - - private ByteBuffer createDocsumPacketData(byte[] docsumData) { - ByteBuffer buffer=ByteBuffer.allocate(docsumData.length+12+4); - buffer.limit(buffer.capacity()); - buffer.position(0); - buffer.putInt(docsumData.length+8+4); - buffer.putInt(205); // Docsum packet code - buffer.putInt(0); - buffer.putInt(0); - buffer.put(docsumData); - return buffer; - } - - } - */ -} -- cgit v1.2.3 From 354136dc94d1963384573d2f2728f1fb408f3d9a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 15:51:38 +0100 Subject: Rename FastBackend => Indexedbackend, and move some tests into the package they test. --- .../vespa/model/container/PlatformBundles.java | 3 +- .../configdefinitions/container.qr-searchers.def | 2 +- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 20 +- .../com/yahoo/prelude/fastsearch/FastBackend.java | 160 ---------------- .../yahoo/prelude/fastsearch/IndexedBackend.java | 160 ++++++++++++++++ .../fastsearch/DocsumDefinitionTestCase.java | 82 +++++++++ .../prelude/fastsearch/IndexedBackendTestCase.java | 199 ++++++++++++++++++++ .../prelude/fastsearch/PartialFillTestCase.java | 140 ++++++++++++++ .../fastsearch/test/DocsumDefinitionTestCase.java | 90 --------- .../fastsearch/test/FastSearcherTestCase.java | 204 --------------------- .../fastsearch/test/PartialFillTestCase.java | 150 --------------- 11 files changed, 594 insertions(+), 616 deletions(-) delete mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java create mode 100644 container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index f1c85699af5..5f363ff993f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.prelude.fastsearch.IndexedBackend; import com.yahoo.vespa.defaults.Defaults; import java.nio.file.Path; @@ -92,7 +93,7 @@ public class PlatformBundles { com.yahoo.docproc.SimpleDocumentProcessor.class.getName(), com.yahoo.language.simple.SimpleLinguistics.class.getName(), com.yahoo.prelude.cluster.ClusterSearcher.class.getName(), - com.yahoo.prelude.fastsearch.FastBackend.class.getName(), + IndexedBackend.class.getName(), com.yahoo.prelude.fastsearch.VespaBackend.class.getName(), com.yahoo.prelude.querytransform.CJKSearcher.class.getName(), com.yahoo.prelude.querytransform.CollapsePhraseSearcher.class.getName(), diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index 2cd4af4fdf1..ac0c0dd6ada 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -25,7 +25,7 @@ com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" ## Default docsum class the QR server should ask the backend to ## use for representing hints as default. -com.yahoo.prelude.fastsearch.FastBackend.docsum.defaultclass string default="" +com.yahoo.prelude.fastsearch.IndexedBackend.docsum.defaultclass string default="" com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 28ba4fb09b2..a8605ab1597 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -11,7 +11,7 @@ import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastBackend; +import com.yahoo.prelude.fastsearch.IndexedBackend; import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; @@ -93,7 +93,7 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); SummaryParameters docSumParams = new SummaryParameters(qrsConfig - .com().yahoo().prelude().fastsearch().FastBackend().docsum() + .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() .defaultclass()); String uniqueServerId = UUID.randomUUID().toString(); @@ -130,20 +130,20 @@ public class ClusterSearcher extends Searcher { return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); } - private static FastBackend searchDispatch(int searchclusterIndex, - String searchClusterName, - String serverId, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - ComponentRegistry dispatchers) { + private static IndexedBackend searchDispatch(int searchclusterIndex, + String searchClusterName, + String serverId, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + ComponentRegistry dispatchers) { ClusterParams clusterParams = makeClusterParams(searchclusterIndex); ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName); Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId); if (dispatcher == null) throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured"); - return new FastBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + return new IndexedBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); } private static StreamingBackend streamingCluster(String serverId, diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java deleted file mode 100644 index b1b3483af8c..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastBackend.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.yahoo.prelude.querytransform.QueryRewrite; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.Dispatcher; -import com.yahoo.search.dispatch.FillInvoker; -import com.yahoo.search.dispatch.SearchInvoker; -import com.yahoo.search.grouping.GroupingRequest; -import com.yahoo.search.grouping.request.GroupingOperation; -import com.yahoo.search.query.Ranking; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.schema.SchemaInfo; - -import java.io.IOException; -import java.util.Optional; - -/** - * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 - * network layer. - * - * @author bratseth - */ -// TODO: Clean up all the duplication in the various search methods by -// switching to doing all the error handling using exceptions below doSearch2. -// Right now half is done by exceptions handled in doSearch2 and half by setting -// errors on results and returning them. It could be handy to create a QueryHandlingErrorException -// or similar which could wrap an error message, and then just always throw that and -// catch and unwrap into a results with an error in high level methods. -Jon -public class FastBackend extends VespaBackend { - - /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ - private final Dispatcher dispatcher; - - /** - * Creates a Fastsearcher. - * - * @param serverId the resource pool used to create direct connections to the local search nodes when - * bypassing the dispatch node - * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. - * Eventually we will move everything to this protocol and never use dispatch nodes. - * At that point we won't need a cluster searcher above this to select and pass the right - * backend. - * @param docSumParams document summary parameters - * @param clusterParams the cluster number, and other cluster backend parameters - * @param documentdbInfoConfig document database parameters - */ - public FastBackend(String serverId, - Dispatcher dispatcher, - SummaryParameters docSumParams, - ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo) { - init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); - this.dispatcher = dispatcher; - } - - @Override - protected void transformQuery(Query query) { - QueryRewrite.rewriteSddocname(query); - } - - private void injectSource(HitGroup hits) { - for (Hit hit : hits.asUnorderedHits()) { - if (hit instanceof FastHit) { - hit.setSource(getName()); - } - } - } - - @Override - public Result doSearch2(String schema, Query query) { - if (dispatcher.allGroupsHaveSize1()) - forceSinglePassGrouping(query); - try (SearchInvoker invoker = getSearchInvoker(query)) { - Result result = invoker.search(query); - injectSource(result.hits()); - - if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { - // There is currently no correct choice for which - // summary class we want to fetch at this point. If we - // fetch the one selected by the user it may not - // contain the data we need. If we fetch the default - // one we end up fetching docsums twice unless the - // user also requested the default one. - fill(result, query.getPresentation().getSummary()); // ARGH - } - return result; - } catch (TimeoutException e) { - return new Result(query,ErrorMessage.createTimeout(e.getMessage())); - } catch (IOException e) { - Result result = new Result(query); - if (query.getTrace().getLevel() >= 1) - query.trace(getName() + " error response: " + result, false, 1); - result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); - return result; - } - } - - /** - * Perform a partial docsum fill for a temporary result - * representing a partition of the complete fill request. - * - * @param result result containing a partition of the unfilled hits - * @param summaryClass the summary class we want to fill with - **/ - @Override - protected void doPartialFill(Result result, String summaryClass) { - if (result.isFilled(summaryClass)) return; - - Query query = result.getQuery(); - traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); - - try (FillInvoker invoker = getFillInvoker(result)) { - invoker.fill(result, summaryClass); - } - } - - /** When we only search a single node, doing all grouping in one pass is more efficient */ - private void forceSinglePassGrouping(Query query) { - for (GroupingRequest groupingRequest : query.getSelect().getGrouping()) - forceSinglePassGrouping(groupingRequest.getRootOperation()); - } - - private void forceSinglePassGrouping(GroupingOperation operation) { - operation.setForceSinglePass(true); - for (GroupingOperation childOperation : operation.getChildren()) - forceSinglePassGrouping(childOperation); - } - - /** - * Returns an invocation object for use in a single search request. The specific implementation returned - * depends on query properties with the default being an invoker that interfaces with a dispatcher - * on the same host. - */ - private SearchInvoker getSearchInvoker(Query query) { - return dispatcher.getSearchInvoker(query, this); - } - - /** - * Returns an invocation object for use in a single fill request. The specific implementation returned - * depends on query properties with the default being an invoker that uses RPC to interface with - * content nodes. - */ - private FillInvoker getFillInvoker(Result result) { - return dispatcher.getFillInvoker(result, this); - } - - private static Optional quotedSummaryClass(String summaryClass) { - return Optional.of(summaryClass == null ? "[null]" : "'" + summaryClass + "'"); - } - - public String toString() { - return "fast searcher (" + getName() + ") "; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java new file mode 100644 index 00000000000..294aff8d78b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java @@ -0,0 +1,160 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.FillInvoker; +import com.yahoo.search.dispatch.SearchInvoker; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.schema.SchemaInfo; + +import java.io.IOException; +import java.util.Optional; + +/** + * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 + * network layer. + * + * @author bratseth + */ +// TODO: Clean up all the duplication in the various search methods by +// switching to doing all the error handling using exceptions below doSearch2. +// Right now half is done by exceptions handled in doSearch2 and half by setting +// errors on results and returning them. It could be handy to create a QueryHandlingErrorException +// or similar which could wrap an error message, and then just always throw that and +// catch and unwrap into a results with an error in high level methods. -Jon +public class IndexedBackend extends VespaBackend { + + /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ + private final Dispatcher dispatcher; + + /** + * Creates a Fastsearcher. + * + * @param serverId the resource pool used to create direct connections to the local search nodes when + * bypassing the dispatch node + * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. + * Eventually we will move everything to this protocol and never use dispatch nodes. + * At that point we won't need a cluster searcher above this to select and pass the right + * backend. + * @param docSumParams document summary parameters + * @param clusterParams the cluster number, and other cluster backend parameters + * @param documentdbInfoConfig document database parameters + */ + public IndexedBackend(String serverId, + Dispatcher dispatcher, + SummaryParameters docSumParams, + ClusterParams clusterParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo) { + init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + this.dispatcher = dispatcher; + } + + @Override + protected void transformQuery(Query query) { + QueryRewrite.rewriteSddocname(query); + } + + private void injectSource(HitGroup hits) { + for (Hit hit : hits.asUnorderedHits()) { + if (hit instanceof FastHit) { + hit.setSource(getName()); + } + } + } + + @Override + public Result doSearch2(String schema, Query query) { + if (dispatcher.allGroupsHaveSize1()) + forceSinglePassGrouping(query); + try (SearchInvoker invoker = getSearchInvoker(query)) { + Result result = invoker.search(query); + injectSource(result.hits()); + + if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { + // There is currently no correct choice for which + // summary class we want to fetch at this point. If we + // fetch the one selected by the user it may not + // contain the data we need. If we fetch the default + // one we end up fetching docsums twice unless the + // user also requested the default one. + fill(result, query.getPresentation().getSummary()); // ARGH + } + return result; + } catch (TimeoutException e) { + return new Result(query,ErrorMessage.createTimeout(e.getMessage())); + } catch (IOException e) { + Result result = new Result(query); + if (query.getTrace().getLevel() >= 1) + query.trace(getName() + " error response: " + result, false, 1); + result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); + return result; + } + } + + /** + * Perform a partial docsum fill for a temporary result + * representing a partition of the complete fill request. + * + * @param result result containing a partition of the unfilled hits + * @param summaryClass the summary class we want to fill with + **/ + @Override + protected void doPartialFill(Result result, String summaryClass) { + if (result.isFilled(summaryClass)) return; + + Query query = result.getQuery(); + traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); + + try (FillInvoker invoker = getFillInvoker(result)) { + invoker.fill(result, summaryClass); + } + } + + /** When we only search a single node, doing all grouping in one pass is more efficient */ + private void forceSinglePassGrouping(Query query) { + for (GroupingRequest groupingRequest : query.getSelect().getGrouping()) + forceSinglePassGrouping(groupingRequest.getRootOperation()); + } + + private void forceSinglePassGrouping(GroupingOperation operation) { + operation.setForceSinglePass(true); + for (GroupingOperation childOperation : operation.getChildren()) + forceSinglePassGrouping(childOperation); + } + + /** + * Returns an invocation object for use in a single search request. The specific implementation returned + * depends on query properties with the default being an invoker that interfaces with a dispatcher + * on the same host. + */ + private SearchInvoker getSearchInvoker(Query query) { + return dispatcher.getSearchInvoker(query, this); + } + + /** + * Returns an invocation object for use in a single fill request. The specific implementation returned + * depends on query properties with the default being an invoker that uses RPC to interface with + * content nodes. + */ + private FillInvoker getFillInvoker(Result result) { + return dispatcher.getFillInvoker(result, this); + } + + private static Optional quotedSummaryClass(String summaryClass) { + return Optional.of(summaryClass == null ? "[null]" : "'" + summaryClass + "'"); + } + + public String toString() { + return "fast searcher (" + getName() + ") "; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java new file mode 100644 index 00000000000..ba9988b865c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java @@ -0,0 +1,82 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.document.GlobalId; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.Schema; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests docsum class functionality + * + * @author bratseth + */ +public class DocsumDefinitionTestCase { + + @Test + void testDecoding() { + DocsumDefinitionSet set = createDocsumDefinitionSet(); + FastHit hit = new FastHit(); + + set.lazyDecode(null, makeDocsum(), hit); + assertEquals("Arts/Celebrities/Madonna", hit.getField("TOPIC")); + assertEquals("1", hit.getField("EXTINFOSOURCE").toString()); + assertEquals("10", hit.getField("LANG1").toString()); + assertEquals("352", hit.getField("WORDS").toString()); + assertEquals("index:null/0/" + asHexString(hit.getGlobalId()), hit.getId().toString()); + } + + private static String asHexString(GlobalId gid) { + StringBuilder sb = new StringBuilder(); + byte[] rawGid = gid.getRawId(); + for (byte b : rawGid) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) + sb.append('0'); + sb.append(hex); + } + return sb.toString(); + } + + public static byte[] makeDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setString("TOPIC", "Arts/Celebrities/Madonna"); + docsum.setString("TITLE", "StudyOfMadonna.com - Interviews, Articles, Reviews, Quotes, Essays and more.."); + docsum.setString("DYNTEASER", "dynamic teaser"); + docsum.setLong("EXTINFOSOURCE", 1); + docsum.setLong("LANG1", 10); + docsum.setLong("WORDS", 352); + docsum.setLong("BYTES", 9190); + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + + public static DocsumDefinitionSet createDocsumDefinitionSet() { + var schema = new Schema.Builder("test"); + var summary = new DocumentSummary.Builder("default"); + summary.add(new DocumentSummary.Field("TOPIC", "string")); + summary.add(new DocumentSummary.Field("TITLE", "string")); + summary.add(new DocumentSummary.Field("DYNTEASER", "string")); + summary.add(new DocumentSummary.Field("EXTINFOSOURCE", "integer")); + summary.add(new DocumentSummary.Field("LANG1", "integer")); + summary.add(new DocumentSummary.Field("WORDS", "integer")); + summary.add(new DocumentSummary.Field("BYTES", "byte")); + schema.add(summary.build()); + return new DocsumDefinitionSet(schema.build()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java new file mode 100644 index 00000000000..917206bf00c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java @@ -0,0 +1,199 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.handler.VipStatus; +import com.yahoo.container.protect.Error; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.dispatch.MockDispatcher; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.junit.jupiter.api.Assertions.*; + + +/** + * Tests the Fast searcher + * + * @author bratseth + */ +public class IndexedBackendTestCase { + private static final String SCHEMA = "test"; + private static final String CLUSTER = "test"; + + @Test + void testNullQuery() { + Logger.getLogger(IndexedBackend.class.getName()).setLevel(Level.ALL); + IndexedBackend fastSearcher = new IndexedBackend("container.0", + MockDispatcher.create(List.of()), + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); + + String query = "?junkparam=ignored"; + Result result = doSearch(fastSearcher, new Query(query), 0, 10); + ErrorMessage message = result.hits().getError(); + + assertNotNull(message, "Got error"); + assertEquals("Null query", message.getMessage()); + assertEquals(query, message.getDetailedMessage()); + assertEquals(Error.NULL_QUERY.code, message.getCode()); + } + + private Result doSearch(VespaBackend searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return searcher.search(SCHEMA, query); + } + + @Test + void testSinglePassGroupingIsForcedWithSingleNodeGroups() { + IndexedBackend fastSearcher = new IndexedBackend("container.0", + MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); + Query q = new Query("?query=foo"); + GroupingRequest request1 = GroupingRequest.newInstance(q); + request1.setRootOperation(new AllOperation()); + + GroupingRequest request2 = GroupingRequest.newInstance(q); + AllOperation all = new AllOperation(); + all.addChild(new EachOperation()); + all.addChild(new EachOperation()); + request2.setRootOperation(all); + + assertForceSinglePassIs(false, q); + fastSearcher.search(SCHEMA, q); + assertForceSinglePassIs(true, q); + } + + @Test + void testRankProfileValidation() { + IndexedBackend fastSearcher = new IndexedBackend("container.0", + MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); + assertFalse(searchError("?query=q", fastSearcher).contains("does not contain requested rank profile")); + assertFalse(searchError("?query=q&ranking.profile=default", fastSearcher).contains("does not contain requested rank profile")); + assertTrue(searchError("?query=q&ranking.profile=nosuch", fastSearcher).contains("does not contain requested rank profile")); + } + + @Test + void testSummaryNeedsQuery() { + var documentDb = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name(SCHEMA))); + var schema = new Schema.Builder(SCHEMA) + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("default").setHasRankFeatures(false) + .setHasSummaryFeatures(false) + .build()); + IndexedBackend backend = new IndexedBackend("container.0", + MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0))), + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentDb, + new SchemaInfo(List.of(schema.build()), List.of())); + Query q = new Query("?query=foo"); + Result result = doSearch(backend, q, 0, 10); + assertFalse(backend.summaryNeedsQuery(q)); + + q = new Query("?query=select+*+from+source+where+title+contains+%22foobar%22+and++geoLocation%28myfieldname%2C+63.5%2C+10.5%2C+%22999+km%22%29%3B"); + q.getModel().setType(Query.Type.YQL); + result = doSearch(backend, q, 0, 10); + assertTrue(backend.summaryNeedsQuery(q)); + } + + @Test + void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { + MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0), new Node(CLUSTER, 2, "host1", 0))); + + IndexedBackend fastSearcher = new IndexedBackend("container.0", + dispatcher, + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); + Query q = new Query("?query=foo"); + GroupingRequest request1 = GroupingRequest.newInstance(q); + request1.setRootOperation(new AllOperation()); + + GroupingRequest request2 = GroupingRequest.newInstance(q); + AllOperation all = new AllOperation(); + all.addChild(new EachOperation()); + all.addChild(new EachOperation()); + request2.setRootOperation(all); + + assertForceSinglePassIs(false, q); + fastSearcher.search(SCHEMA, q); + assertForceSinglePassIs(false, q); + } + + private void assertForceSinglePassIs(boolean expected, Query query) { + for (GroupingRequest request : query.getSelect().getGrouping()) + assertForceSinglePassIs(expected, request.getRootOperation()); + } + + private void assertForceSinglePassIs(boolean expected, GroupingOperation operation) { + assertEquals(expected, operation.getForceSinglePass(), "Force single pass is " + expected + " in " + operation); + for (GroupingOperation child : operation.getChildren()) + assertForceSinglePassIs(expected, child); + } + + @Test + void testDispatchReconfig() { + String clusterName = "a"; + var b = new QrSearchersConfig.Builder(); + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + searchClusterB.name(clusterName); + b.searchcluster(searchClusterB); + VipStatus vipStatus = new VipStatus(b.build()); + List nodes_1 = List.of(new Node(CLUSTER, 0, "host0", 0)); + RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(), MockDispatcher.toNodesConfig(nodes_1)); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); + dispatch_1.clusterMonitor.shutdown(); + vipStatus.addToRotation(clusterName); + assertTrue(vipStatus.isInRotation()); + dispatch_1.deconstruct(); + assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus + } + + private String searchError(String query, VespaBackend searcher) { + return search(query, searcher).hits().getError().getDetailedMessage(); + } + + private Result search(String query, VespaBackend searcher) { + return searcher.search(SCHEMA, new Query(query)); + } + + private DocumentdbInfoConfig documentdbInfoConfig(String schemaName) { + var db = new DocumentdbInfoConfig.Documentdb.Builder().name(schemaName); + return new DocumentdbInfoConfig.Builder().documentdb(db).build(); + } + + private SchemaInfo schemaInfo(String schemaName) { + var schema = new Schema.Builder(schemaName); + schema.add(new RankProfile.Builder("default").build()); + return new SchemaInfo(List.of(schema.build()), List.of()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java new file mode 100644 index 00000000000..7760e204d4b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java @@ -0,0 +1,140 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author havardpe + */ +public class PartialFillTestCase { + + public static class FS4 extends VespaBackend { + public List history = new ArrayList<>(); + protected Result doSearch2(String schema, Query query) { + return new Result(query); + } + protected void doPartialFill(Result result, String summaryClass) { + history.add(result); + } + } + + public static class BadFS4 extends VespaBackend { + protected Result doSearch2(String schema, Query query) { + return new Result(query); + } + protected void doPartialFill(Result result, String summaryClass) { + if (result.hits().getErrorHit() == null) { + result.hits().addError(ErrorMessage.createUnspecifiedError("error")); + } + } + } + + @Test + void testPartitioning() { + FS4 fs4 = new FS4(); + Query a = new Query("/?query=foo"); + Query b = new Query("/?query=bar"); + Query c = new Query("/?query=foo"); // equal to a + Result r = new Result(new Query("/?query=ignorethis")); + for (int i = 0; i < 7; i++) { + FastHit h = new FastHit(); + h.setQuery(a); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + h.setQuery(b); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 3; i++) { + FastHit h = new FastHit(); + h.setQuery(c); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 2; i++) { + FastHit h = new FastHit(); + // no query assigned + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + // not fillable + h.setQuery(a); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + // already filled + h.setQuery(a); + h.setFilled("default"); + r.hits().add(h); + } + doFill(fs4, r, "default"); + assertNull(r.hits().getErrorHit()); + assertEquals(4, fs4.history.size()); + assertEquals(a, fs4.history.get(0).getQuery()); + assertEquals(7, fs4.history.get(0).getHitCount()); + assertEquals(b, fs4.history.get(1).getQuery()); + assertEquals(5, fs4.history.get(1).getHitCount()); + assertEquals(c, fs4.history.get(2).getQuery()); + assertEquals(3, fs4.history.get(2).getHitCount()); + assertEquals(r.getQuery(), fs4.history.get(3).getQuery()); + assertEquals(2, fs4.history.get(3).getHitCount()); + } + + @Test + void testMergeErrors() { + BadFS4 fs4 = new BadFS4(); + Query a = new Query("/?query=foo"); + Query b = new Query("/?query=bar"); + Result r = new Result(new Query("/?query=ignorethis")); + { + FastHit h = new FastHit(); + h.setQuery(a); + h.setFillable(); + r.hits().add(h); + } + { + FastHit h = new FastHit(); + h.setQuery(b); + h.setFillable(); + r.hits().add(h); + } + doFill(fs4, r, "default"); + ErrorHit eh = r.hits().getErrorHit(); + assertNotNull(eh); + ErrorMessage exp_sub = ErrorMessage.createUnspecifiedError("error"); + int n = 0; + for (Iterator i = eh.errorIterator(); i.hasNext(); ) { + com.yahoo.search.result.ErrorMessage error = i.next(); + switch (n) { + case 0: + case 1: + assertEquals(exp_sub, error); + break; + default: + fail(); + } + n++; + } + } + + private void doFill(VespaBackend searcher, Result result, String summaryClass) { + searcher.fill(result, summaryClass); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java deleted file mode 100644 index ade094115fe..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; - -import com.yahoo.prelude.fastsearch.ByteField; -import com.yahoo.prelude.fastsearch.DataField; -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.IntegerField; -import com.yahoo.prelude.fastsearch.StringField; -import com.yahoo.document.DocumentId; -import com.yahoo.document.GlobalId; -import com.yahoo.search.schema.DocumentSummary; -import com.yahoo.search.schema.Schema; -import com.yahoo.slime.BinaryFormat; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * Tests docsum class functionality - * - * @author bratseth - */ -public class DocsumDefinitionTestCase { - - @Test - void testDecoding() { - DocsumDefinitionSet set = createDocsumDefinitionSet(); - FastHit hit = new FastHit(); - - set.lazyDecode(null, makeDocsum(), hit); - assertEquals("Arts/Celebrities/Madonna", hit.getField("TOPIC")); - assertEquals("1", hit.getField("EXTINFOSOURCE").toString()); - assertEquals("10", hit.getField("LANG1").toString()); - assertEquals("352", hit.getField("WORDS").toString()); - assertEquals("index:null/0/" + asHexString(hit.getGlobalId()), hit.getId().toString()); - } - - private static String asHexString(GlobalId gid) { - StringBuilder sb = new StringBuilder(); - byte[] rawGid = gid.getRawId(); - for (byte b : rawGid) { - String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) - sb.append('0'); - sb.append(hex); - } - return sb.toString(); - } - - public static byte[] makeDocsum() { - Slime slime = new Slime(); - Cursor docsum = slime.setObject(); - docsum.setString("TOPIC", "Arts/Celebrities/Madonna"); - docsum.setString("TITLE", "StudyOfMadonna.com - Interviews, Articles, Reviews, Quotes, Essays and more.."); - docsum.setString("DYNTEASER", "dynamic teaser"); - docsum.setLong("EXTINFOSOURCE", 1); - docsum.setLong("LANG1", 10); - docsum.setLong("WORDS", 352); - docsum.setLong("BYTES", 9190); - byte[] tmp = BinaryFormat.encode(slime); - ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); - buf.order(ByteOrder.BIG_ENDIAN); - buf.put(tmp); - return buf.array(); - } - - public static DocsumDefinitionSet createDocsumDefinitionSet() { - var schema = new Schema.Builder("test"); - var summary = new DocumentSummary.Builder("default"); - summary.add(new DocumentSummary.Field("TOPIC", "string")); - summary.add(new DocumentSummary.Field("TITLE", "string")); - summary.add(new DocumentSummary.Field("DYNTEASER", "string")); - summary.add(new DocumentSummary.Field("EXTINFOSOURCE", "integer")); - summary.add(new DocumentSummary.Field("LANG1", "integer")); - summary.add(new DocumentSummary.Field("WORDS", "integer")); - summary.add(new DocumentSummary.Field("BYTES", "byte")); - schema.add(summary.build()); - return new DocsumDefinitionSet(schema.build()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java deleted file mode 100644 index 3928743c954..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; - -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.handler.VipStatus; -import com.yahoo.container.protect.Error; -import com.yahoo.prelude.fastsearch.ClusterParams; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastBackend; -import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackend; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.MockDispatcher; -import com.yahoo.search.dispatch.rpc.RpcResourcePool; -import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.grouping.GroupingRequest; -import com.yahoo.search.grouping.request.AllOperation; -import com.yahoo.search.grouping.request.EachOperation; -import com.yahoo.search.grouping.request.GroupingOperation; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.schema.DocumentSummary; -import com.yahoo.search.schema.RankProfile; -import com.yahoo.search.schema.Schema; -import com.yahoo.search.schema.SchemaInfo; -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.junit.jupiter.api.Assertions.*; - - -/** - * Tests the Fast searcher - * - * @author bratseth - */ -public class FastSearcherTestCase { - private static final String SCHEMA = "test"; - private static final String CLUSTER = "test"; - - @Test - void testNullQuery() { - Logger.getLogger(FastBackend.class.getName()).setLevel(Level.ALL); - FastBackend fastSearcher = new FastBackend("container.0", - MockDispatcher.create(List.of()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig(SCHEMA), - schemaInfo(SCHEMA)); - - String query = "?junkparam=ignored"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - ErrorMessage message = result.hits().getError(); - - assertNotNull(message, "Got error"); - assertEquals("Null query", message.getMessage()); - assertEquals(query, message.getDetailedMessage()); - assertEquals(Error.NULL_QUERY.code, message.getCode()); - } - - private Result doSearch(VespaBackend searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return searcher.search(SCHEMA, query); - } - - @Test - void testSinglePassGroupingIsForcedWithSingleNodeGroups() { - FastBackend fastSearcher = new FastBackend("container.0", - MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig(SCHEMA), - schemaInfo(SCHEMA)); - Query q = new Query("?query=foo"); - GroupingRequest request1 = GroupingRequest.newInstance(q); - request1.setRootOperation(new AllOperation()); - - GroupingRequest request2 = GroupingRequest.newInstance(q); - AllOperation all = new AllOperation(); - all.addChild(new EachOperation()); - all.addChild(new EachOperation()); - request2.setRootOperation(all); - - assertForceSinglePassIs(false, q); - fastSearcher.search(SCHEMA, q); - assertForceSinglePassIs(true, q); - } - - @Test - void testRankProfileValidation() { - FastBackend fastSearcher = new FastBackend("container.0", - MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig(SCHEMA), - schemaInfo(SCHEMA)); - assertFalse(searchError("?query=q", fastSearcher).contains("does not contain requested rank profile")); - assertFalse(searchError("?query=q&ranking.profile=default", fastSearcher).contains("does not contain requested rank profile")); - assertTrue(searchError("?query=q&ranking.profile=nosuch", fastSearcher).contains("does not contain requested rank profile")); - } - - @Test - void testSummaryNeedsQuery() { - var documentDb = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name(SCHEMA))); - var schema = new Schema.Builder(SCHEMA) - .add(new DocumentSummary.Builder("default").build()) - .add(new RankProfile.Builder("default").setHasRankFeatures(false) - .setHasSummaryFeatures(false) - .build()); - FastBackend backend = new FastBackend("container.0", - MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0))), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentDb, - new SchemaInfo(List.of(schema.build()), List.of())); - Query q = new Query("?query=foo"); - Result result = doSearch(backend, q, 0, 10); - assertFalse(backend.summaryNeedsQuery(q)); - - q = new Query("?query=select+*+from+source+where+title+contains+%22foobar%22+and++geoLocation%28myfieldname%2C+63.5%2C+10.5%2C+%22999+km%22%29%3B"); - q.getModel().setType(Query.Type.YQL); - result = doSearch(backend, q, 0, 10); - assertTrue(backend.summaryNeedsQuery(q)); - } - - @Test - void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0), new Node(CLUSTER, 2, "host1", 0))); - - FastBackend fastSearcher = new FastBackend("container.0", - dispatcher, - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig(SCHEMA), - schemaInfo(SCHEMA)); - Query q = new Query("?query=foo"); - GroupingRequest request1 = GroupingRequest.newInstance(q); - request1.setRootOperation(new AllOperation()); - - GroupingRequest request2 = GroupingRequest.newInstance(q); - AllOperation all = new AllOperation(); - all.addChild(new EachOperation()); - all.addChild(new EachOperation()); - request2.setRootOperation(all); - - assertForceSinglePassIs(false, q); - fastSearcher.search(SCHEMA, q); - assertForceSinglePassIs(false, q); - } - - private void assertForceSinglePassIs(boolean expected, Query query) { - for (GroupingRequest request : query.getSelect().getGrouping()) - assertForceSinglePassIs(expected, request.getRootOperation()); - } - - private void assertForceSinglePassIs(boolean expected, GroupingOperation operation) { - assertEquals(expected, operation.getForceSinglePass(), "Force single pass is " + expected + " in " + operation); - for (GroupingOperation child : operation.getChildren()) - assertForceSinglePassIs(expected, child); - } - - @Test - void testDispatchReconfig() { - String clusterName = "a"; - var b = new QrSearchersConfig.Builder(); - var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name(clusterName); - b.searchcluster(searchClusterB); - VipStatus vipStatus = new VipStatus(b.build()); - List nodes_1 = List.of(new Node(CLUSTER, 0, "host0", 0)); - RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(), MockDispatcher.toNodesConfig(nodes_1)); - MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); - dispatch_1.clusterMonitor.shutdown(); - vipStatus.addToRotation(clusterName); - assertTrue(vipStatus.isInRotation()); - dispatch_1.deconstruct(); - assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus - } - - private String searchError(String query, VespaBackend searcher) { - return search(query, searcher).hits().getError().getDetailedMessage(); - } - - private Result search(String query, VespaBackend searcher) { - return searcher.search(SCHEMA, new Query(query)); - } - - private DocumentdbInfoConfig documentdbInfoConfig(String schemaName) { - var db = new DocumentdbInfoConfig.Documentdb.Builder().name(schemaName); - return new DocumentdbInfoConfig.Builder().documentdb(db).build(); - } - - private SchemaInfo schemaInfo(String schemaName) { - var schema = new Schema.Builder(schemaName); - schema.add(new RankProfile.Builder("default").build()); - return new SchemaInfo(List.of(schema.build()), List.of()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java deleted file mode 100644 index cf5a0ca7bd9..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.VespaBackend; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author havardpe - */ -public class PartialFillTestCase { - - public static class FS4 extends VespaBackend { - public List history = new ArrayList<>(); - protected Result doSearch2(String schema, Query query) { - return new Result(query); - } - protected void doPartialFill(Result result, String summaryClass) { - history.add(result); - } - } - - public static class BadFS4 extends VespaBackend { - protected Result doSearch2(String schema, Query query) { - return new Result(query); - } - protected void doPartialFill(Result result, String summaryClass) { - if (result.hits().getErrorHit() == null) { - result.hits().addError(ErrorMessage.createUnspecifiedError("error")); - } - } - } - - @Test - void testPartitioning() { - FS4 fs4 = new FS4(); - Query a = new Query("/?query=foo"); - Query b = new Query("/?query=bar"); - Query c = new Query("/?query=foo"); // equal to a - Result r = new Result(new Query("/?query=ignorethis")); - for (int i = 0; i < 7; i++) { - FastHit h = new FastHit(); - h.setQuery(a); - h.setFillable(); - r.hits().add(h); - } - for (int i = 0; i < 5; i++) { - FastHit h = new FastHit(); - h.setQuery(b); - h.setFillable(); - r.hits().add(h); - } - for (int i = 0; i < 3; i++) { - FastHit h = new FastHit(); - h.setQuery(c); - h.setFillable(); - r.hits().add(h); - } - for (int i = 0; i < 2; i++) { - FastHit h = new FastHit(); - // no query assigned - h.setFillable(); - r.hits().add(h); - } - for (int i = 0; i < 5; i++) { - FastHit h = new FastHit(); - // not fillable - h.setQuery(a); - r.hits().add(h); - } - for (int i = 0; i < 5; i++) { - FastHit h = new FastHit(); - // already filled - h.setQuery(a); - h.setFilled("default"); - r.hits().add(h); - } - doFill(fs4, r, "default"); - assertNull(r.hits().getErrorHit()); - assertEquals(4, fs4.history.size()); - assertEquals(a, fs4.history.get(0).getQuery()); - assertEquals(7, fs4.history.get(0).getHitCount()); - assertEquals(b, fs4.history.get(1).getQuery()); - assertEquals(5, fs4.history.get(1).getHitCount()); - assertEquals(c, fs4.history.get(2).getQuery()); - assertEquals(3, fs4.history.get(2).getHitCount()); - assertEquals(r.getQuery(), fs4.history.get(3).getQuery()); - assertEquals(2, fs4.history.get(3).getHitCount()); - } - - @Test - void testMergeErrors() { - BadFS4 fs4 = new BadFS4(); - Query a = new Query("/?query=foo"); - Query b = new Query("/?query=bar"); - Result r = new Result(new Query("/?query=ignorethis")); - { - FastHit h = new FastHit(); - h.setQuery(a); - h.setFillable(); - r.hits().add(h); - } - { - FastHit h = new FastHit(); - h.setQuery(b); - h.setFillable(); - r.hits().add(h); - } - doFill(fs4, r, "default"); - ErrorHit eh = r.hits().getErrorHit(); - assertNotNull(eh); - ErrorMessage exp_sub = ErrorMessage.createUnspecifiedError("error"); - int n = 0; - for (Iterator i = eh.errorIterator(); i.hasNext(); ) { - com.yahoo.search.result.ErrorMessage error = i.next(); - switch (n) { - case 0: - case 1: - assertEquals(exp_sub, error); - break; - default: - fail(); - } - n++; - } - } - - private void doFill(VespaBackend searcher, Result result, String summaryClass) { - searcher.fill(result, summaryClass); - } - - private Chain chainedAsSearchChain(Searcher topOfChain) { - List searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} -- cgit v1.2.3 From de954d574001b29ea15b55f9b327ea240ca32892 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 11 Mar 2024 16:32:20 +0100 Subject: Test that multiple backends can be used. --- .../com/yahoo/prelude/fastsearch/BoolField.java | 1 - .../prelude/cluster/ClusterSearcherTestCase.java | 30 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java index c0f42f6924b..6a980c37f4e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -7,7 +7,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.search.result.NanNumber; /** * @author bratseth diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index d52fbe580a7..ff91f346195 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -130,6 +130,29 @@ public class ClusterSearcherTestCase { } } + @Test + void testThatMultipleBackendsAreUsed() { + var backendA = new MyMockBackend(false); + var backendB = new MyMockBackend(false); + SchemaInfo schemaInfo = createSchemaInfo(); + var cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backendA, "type2", backendB, "type3", backendA), + new InThreadExecutorService()); + try { + Execution execution = new Execution(cluster1, Execution.Context.createContextStub()); + execution.search(new Query("?query=hello")); + assertEquals(2, backendA.queries().size()); + assertEquals(1, backendB.queries().size()); + execution.search(new Query("?query=hello&restrict=type1")); + assertEquals(3, backendA.queries().size()); + assertEquals(1, backendB.queries().size()); + execution.search(new Query("?query=hello&restrict=type2,type3")); + assertEquals(4, backendA.queries().size()); + assertEquals(2, backendB.queries().size()); + } finally { + cluster1.deconstruct(); + } + } + private static class MyMockBackend extends VespaBackend { private final String type1 = "type1"; @@ -137,6 +160,8 @@ public class ClusterSearcherTestCase { private final String type3 = "type3"; private final Map> results = new LinkedHashMap<>(); private final boolean expectAttributePrefetch; + + private final List queries = new ArrayList<>(); static final String ATTRIBUTE_PREFETCH = "attributeprefetch"; private String getId(String type, int i) { @@ -197,8 +222,11 @@ public class ClusterSearcherTestCase { return null; // search() is overriden, this should never be called } + List queries() { return queries; } + @Override public com.yahoo.search.Result search(String schema, Query query) { + queries.add(query); com.yahoo.search.Result result = new com.yahoo.search.Result(query); List hits = getHits(query); if (hits != null) { @@ -419,7 +447,7 @@ public class ClusterSearcherTestCase { assertResult(6, List.of(7.0, 9.0), getResult(3, 2, extra, ex)); assertResult(6, List.of(9.0, 10.0), getResult(4, 2, extra, ex)); assertResult(6, List.of(10.0), getResult(5, 2, extra, ex)); - assertResult(6, List.of(), getResult(6, 2, extra, ex)); + assertResult(6, List.of(), getResult(6, 2, extra, ex)); } private static ClusterSearcher createSearcher(String clusterName, Double maxQueryTimeout, Double maxQueryCacheTimeout, -- cgit v1.2.3