diff options
50 files changed, 537 insertions, 681 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index e303e878550..b35918b3460 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -28,6 +28,7 @@ import java.io.Writer; public class DerivedConfiguration { private final Schema schema; + private final boolean isStreaming; private Summaries summaries; private Juniperrc juniperrc; private AttributeFields attributeFields; @@ -67,6 +68,7 @@ public class DerivedConfiguration { * schema is later modified. */ public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { + this.isStreaming = isStreaming; try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -150,6 +152,10 @@ public class DerivedConfiguration { } } + public boolean isStreaming() { + return isStreaming; + } + public Summaries getSummaries() { return summaries; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java index a05450aa3a0..816e5a074f6 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.logging.Level; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 7e9d79fc858..26107ad2dca 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java index e425f81c0b4..42457295895 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; /** * Class that processes reference fields and removes attribute aspect of such fields from summary. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 13e6ee6684d..5f363ff993f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.prelude.fastsearch.IndexedBackend; import com.yahoo.vespa.defaults.Defaults; import java.nio.file.Path; @@ -92,8 +93,8 @@ public class PlatformBundles { com.yahoo.docproc.SimpleDocumentProcessor.class.getName(), com.yahoo.language.simple.SimpleLinguistics.class.getName(), com.yahoo.prelude.cluster.ClusterSearcher.class.getName(), - com.yahoo.prelude.fastsearch.FastSearcher.class.getName(), - com.yahoo.prelude.fastsearch.VespaBackEndSearcher.class.getName(), + IndexedBackend.class.getName(), + com.yahoo.prelude.fastsearch.VespaBackend.class.getName(), com.yahoo.prelude.querytransform.CJKSearcher.class.getName(), com.yahoo.prelude.querytransform.CollapsePhraseSearcher.class.getName(), com.yahoo.prelude.querytransform.LiteralBoostSearcher.class.getName(), @@ -144,7 +145,7 @@ public class PlatformBundles { com.yahoo.search.searchers.CacheControlSearcher.class.getName(), com.yahoo.search.searchers.RateLimitingSearcher.class.getName(), com.yahoo.vespa.streamingvisitors.MetricsSearcher.class.getName(), - com.yahoo.vespa.streamingvisitors.StreamingSearcher.class.getName() + com.yahoo.vespa.streamingvisitors.StreamingBackend.class.getName() ); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index c0aec7ba0b4..f19af8b62ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -113,8 +113,11 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { for (DocumentDatabase db : documentDbs) { - DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); - docDb.name(db.getName()); + var docDb = new DocumentdbInfoConfig.Documentdb.Builder() + .name(db.getName()) + .mode(db.getDerivedConfiguration().isStreaming() + ? DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING + : DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX); builder.documentdb(docDb); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 5b501bad876..eb4ec1af157 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -289,8 +289,12 @@ public class DocumentDatabaseTestCase { { // documentdb-info config DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, searcherId); assertEquals(2, dcfg.documentdb().size()); - assertEquals("type1", dcfg.documentdb(0).name()); - assertEquals("type2", dcfg.documentdb(1).name()); + var db = dcfg.documentdb(0); + assertEquals("type1", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); + db = dcfg.documentdb(1); + assertEquals("type2", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); } { // attributes config AttributesConfig acfg = model.getConfig(AttributesConfig.class, searcherId); @@ -312,6 +316,7 @@ public class DocumentDatabaseTestCase { assertEquals(1, dcfg.documentdb().size()); DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); assertEquals("type", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); } @Test @@ -319,6 +324,24 @@ public class DocumentDatabaseTestCase { assertDocumentDBConfigAvailableForStreaming("streaming"); } + @Test + void testMixedModeCluster() { + // Will soon change + List<DocType> sds = List.of(DocType.create("a", "index"), DocType.create("b", "streaming")); + var tester = new SchemaTester(); + var model = tester.createModel(sds, ""); + DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); + assertEquals(1, indexed_cfg.documentdb().size()); + var db = indexed_cfg.documentdb(0); + assertEquals("a", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); + DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b"); + assertEquals(1, streaming_cfg.documentdb().size()); + db = streaming_cfg.documentdb(0); + assertEquals("b", db.name()); + assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); + } + private void assertAttributesConfigIndependentOfMode(String mode, List<String> sds, List<String> documentDBConfigIds, Map<String, List<String>> expectedAttributesMap) diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index f8871e9f315..ac0c0dd6ada 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -25,7 +25,7 @@ com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" ## Default docsum class the QR server should ask the backend to ## use for representing hints as default. -com.yahoo.prelude.fastsearch.FastSearcher.docsum.defaultclass string default="" +com.yahoo.prelude.fastsearch.IndexedBackend.docsum.defaultclass string default="" com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1fe0835306e..a8605ab1597 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; +import com.yahoo.collections.TinyIdentitySet; import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; @@ -10,9 +11,9 @@ import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; +import com.yahoo.prelude.fastsearch.IndexedBackend; import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -24,13 +25,14 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.streamingvisitors.StreamingSearcher; +import com.yahoo.vespa.streamingvisitors.StreamingBackend; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -42,8 +44,6 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; import java.util.stream.Collectors; -import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; - /** * A searcher which forwards to a cluster of monitored native Vespa backends. * @@ -60,13 +60,12 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; // The set of document types contained in this search cluster - private final Set<String> schemas; + private final Map<String, VespaBackend> schema2Searcher; private final SchemaInfo schemaInfo; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds - private final VespaBackEndSearcher server; private final Executor executor; private final GlobalPhaseRanker globalPhaseRanker; @@ -88,26 +87,32 @@ public class ClusterSearcher extends Searcher { searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = globalPhaseRanker; - schemas = new LinkedHashSet<>(); + schema2Searcher = new LinkedHashMap<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); SummaryParameters docSumParams = new SummaryParameters(qrsConfig - .com().yahoo().prelude().fastsearch().FastSearcher().docsum() + .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() .defaultclass()); - for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) - schemas.add(docDb.name()); - String uniqueServerId = UUID.randomUUID().toString(); - if (searchClusterConfig.indexingmode() == STREAMING) { - server = streamingCluster(uniqueServerId, searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); - vipStatus.addToRotation(server.getName()); - } else { - server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, - docSumParams, documentDbConfig, schemaInfo, dispatchers); + VespaBackend streaming = null, indexed = null; + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { + if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { + if (indexed == null) { + indexed = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, + docSumParams, documentDbConfig, schemaInfo, dispatchers); + } + schema2Searcher.put(docDb.name(), indexed); + } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) { + if (streaming == null) { + streaming = streamingCluster(uniqueServerId, searchClusterIndex, + searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); + vipStatus.addToRotation(streaming.getName()); + } + schema2Searcher.put(docDb.name(), streaming); + } } } @@ -117,41 +122,42 @@ public class ClusterSearcher extends Searcher { return searchCluster; } } - return null; + throw new IllegalStateException("No configured search cluster '" + name + "' among : " + + config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList()); } private static ClusterParams makeClusterParams(int searchclusterIndex) { return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); } - private static FastSearcher searchDispatch(int searchclusterIndex, - String searchClusterName, - String serverId, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - ComponentRegistry<Dispatcher> dispatchers) { + private static IndexedBackend searchDispatch(int searchclusterIndex, + String searchClusterName, + String serverId, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + ComponentRegistry<Dispatcher> dispatchers) { ClusterParams clusterParams = makeClusterParams(searchclusterIndex); ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName); Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId); if (dispatcher == null) throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured"); - return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + return new IndexedBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); } - private static StreamingSearcher streamingCluster(String serverId, - int searchclusterIndex, - QrSearchersConfig.Searchcluster searchClusterConfig, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - VespaDocumentAccess access) { + private static StreamingBackend streamingCluster(String serverId, + int searchclusterIndex, + QrSearchersConfig.Searchcluster searchClusterConfig, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + VespaDocumentAccess access) { if (searchClusterConfig.searchdef().size() != 1) throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " + searchClusterConfig.searchdef()); ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - StreamingSearcher searcher = new StreamingSearcher(access); + StreamingBackend searcher = new StreamingBackend(access); searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid()); searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); @@ -159,49 +165,57 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { + ClusterSearcher(SchemaInfo schemaInfo, Map<String, VespaBackend> schema2Searcher, Executor executor) { this.schemaInfo = schemaInfo; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; - server = searcher; this.executor = executor; this.globalPhaseRanker = null; - this.schemas = schemas; + this.schema2Searcher = schema2Searcher; } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas) { - this(schemaInfo, schemas, null, null); + ClusterSearcher(SchemaInfo schemaInfo, Map<String, VespaBackend> schema2Searcher) { + this(schemaInfo, schema2Searcher, null); } @Override public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - var searcher = server; - if (searcher == null) { + if (schema2Searcher.isEmpty()) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } if (query.getTimeLeft() <= 0) { return new Result(query, ErrorMessage.createTimeout("No time left for searching")); } - return doSearch(searcher, query); + return doSearch(query); } @Override - public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { + public void fill(Result result, String summaryClass, Execution execution) { + fill(result, summaryClass); + } + private void fill(Result result, String summaryClass) { Query query = result.getQuery(); - - VespaBackEndSearcher searcher = server; - if (searcher != null) { - if (query.getTimeLeft() > 0) { - searcher.fill(result, summaryClass); - } else { - if (result.hits().getErrorHit() == null) { - result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + - query.getTimeout() + " ms")); + var restrict = query.getModel().getRestrict(); + Collection<VespaBackend> servers = (restrict != null && ! restrict.isEmpty()) + ? query.getModel().getRestrict().stream() + .map(schema2Searcher::get) + .collect(Collectors.toCollection(TinyIdentitySet::new)) + : schema2Searcher.values().stream().collect(Collectors.toCollection(TinyIdentitySet::new)); + + if ( ! servers.isEmpty() ) { + for (var server : servers) { + if (query.getTimeLeft() > 0) { + server.fill(result, summaryClass); + } else { + if (result.hits().getErrorHit() == null) { + result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + + query.getTimeout() + " ms")); + } } } } else { @@ -232,17 +246,17 @@ public class ClusterSearcher extends Searcher { query.getRanking().setQueryCache(false); } - private Result doSearch(VespaBackEndSearcher searcher, Query query) { - if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query); + private Result doSearch(Query query) { + if (schema2Searcher.size() > 1) { + return searchMultipleDocumentTypes(query); } else { - String schema = schemas.iterator().next(); + String schema = schema2Searcher.keySet().iterator().next(); query.getModel().setRestrict(schema); - return perSchemaSearch(searcher, schema, query); + return perSchemaSearch(schema, query); } } - private Result perSchemaSearch(VespaBackEndSearcher searcher, String schema, Query query) { + private Result perSchemaSearch(String schema, Query query) { Set<String> restrict = query.getModel().getRestrict(); if (restrict.size() != 1) { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); @@ -258,7 +272,7 @@ public class ClusterSearcher extends Searcher { query.setOffset(0); query.setHits(useHits); } - Result result = searcher.search(schema, query); + Result result = schema2Searcher.get(schema).search(schema, query); if (useGlobalPhase) { globalPhaseRanker.rerankHits(query, result, schema); result.hits().trim(wantOffset, wantHits); @@ -285,17 +299,17 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(VespaBackEndSearcher searcher, Query query) { + private Result searchMultipleDocumentTypes(Query query) { Set<String> schemas = resolveSchemas(query); Map<String, Query> schemaQueries = createQueries(query, schemas); if (schemaQueries.size() == 1) { var entry = schemaQueries.entrySet().iterator().next(); - return perSchemaSearch(searcher, entry.getKey(), entry.getValue()); + return perSchemaSearch(entry.getKey(), entry.getValue()); } else { Result mergedResult = new Result(query); List<FutureTask<Result>> pending = new ArrayList<>(schemaQueries.size()); for (var entry : schemaQueries.entrySet()) { - FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, entry.getKey(), entry.getValue())); + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(entry.getKey(), entry.getValue())); try { executor.execute(task); pending.add(task); @@ -311,7 +325,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(mergedResult, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim @@ -328,7 +342,7 @@ public class ClusterSearcher extends Searcher { candidates.addAll(cluster.schemas()); } return (candidates.isEmpty() ? sources : candidates).stream() - .filter(schemas::contains).collect(Collectors.toUnmodifiableSet()); + .filter(schema2Searcher::containsKey).collect(Collectors.toUnmodifiableSet()); } Set<String> resolveSchemas(Query query) { @@ -336,7 +350,7 @@ public class ClusterSearcher extends Searcher { if (restrict == null || restrict.isEmpty()) { Set<String> sources = query.getModel().getSources(); return (sources == null || sources.isEmpty()) - ? schemas + ? schema2Searcher.keySet() : resolveSourceSubset(sources); } else { return filterValidDocumentTypes(restrict); @@ -346,7 +360,7 @@ public class ClusterSearcher extends Searcher { private Set<String> filterValidDocumentTypes(Collection<String> restrict) { Set<String> retval = new LinkedHashSet<>(); for (String docType : restrict) { - if (docType != null && schemas.contains(docType)) { + if (docType != null && schema2Searcher.containsKey(docType)) { retval.add(docType); } } @@ -375,7 +389,11 @@ public class ClusterSearcher extends Searcher { @Override public void deconstruct() { - if (server != null) { + Map<String, VespaBackend> servers = new HashMap<>(); + for (var server : schema2Searcher.values()) { + servers.put(server.getName(), server); + } + for (var server : servers.values()) { server.shutDown(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java index c0f42f6924b..6a980c37f4e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -7,7 +7,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.search.result.NanNumber; /** * @author bratseth diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java index 85b7770a8b2..294aff8d78b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java @@ -30,7 +30,7 @@ import java.util.Optional; // errors on results and returning them. It could be handy to create a QueryHandlingErrorException // or similar which could wrap an error message, and then just always throw that and // catch and unwrap into a results with an error in high level methods. -Jon -public class FastSearcher extends VespaBackEndSearcher { +public class IndexedBackend extends VespaBackend { /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ private final Dispatcher dispatcher; @@ -48,12 +48,12 @@ public class FastSearcher extends VespaBackEndSearcher { * @param clusterParams the cluster number, and other cluster backend parameters * @param documentdbInfoConfig document database parameters */ - public FastSearcher(String serverId, - Dispatcher dispatcher, - SummaryParameters docSumParams, - ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo) { + public IndexedBackend(String serverId, + Dispatcher dispatcher, + SummaryParameters docSumParams, + ClusterParams clusterParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo) { init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); this.dispatcher = dispatcher; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java index 340262ed186..5191bfc4f41 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -31,7 +31,7 @@ import java.util.Optional; * * @author baldersheim */ -public abstract class VespaBackEndSearcher { +public abstract class VespaBackend { /** for vespa-internal use only; consider renaming the summary class */ public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; @@ -184,7 +184,7 @@ public abstract class VespaBackEndSearcher { return result; } - private List<Result> partitionHits(Result result, String summaryClass) { + private static List<Result> partitionHits(Result result, String summaryClass) { List<Result> parts = new ArrayList<>(); TinyIdentitySet<Query> queryMap = new TinyIdentitySet<>(4); @@ -210,6 +210,7 @@ public abstract class VespaBackEndSearcher { return parts; } + //TODO Add schema here too. public void fill(Result result, String summaryClass) { if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 1128ad9570d..634a16d1dee 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,7 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -115,7 +115,7 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { // TODO: remove or rename this internal summary class for Vespa 9 - execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + execution.fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 8ec88bd7f45..1689f6d246e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -6,7 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.annotation.Inject; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -256,13 +256,13 @@ public class Dispatcher extends AbstractComponent { } } - public FillInvoker getFillInvoker(Result result, VespaBackEndSearcher searcher) { + public FillInvoker getFillInvoker(Result result, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. return items.register(items.get().invokerFactory.createFillInvoker(searcher, result)); } } - public SearchInvoker getSearchInvoker(Query query, VespaBackEndSearcher searcher) { + public SearchInvoker getSearchInvoker(Query query, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. int maxHitsPerNode = dispatchConfig.maxHitsPerNode(); SearchInvoker invoker = getSearchPathInvoker(query, searcher, searchCluster.groupList(), items.get().invokerFactory, maxHitsPerNode) @@ -277,7 +277,7 @@ public class Dispatcher extends AbstractComponent { } /** Builds an invoker based on searchpath */ - private static Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackEndSearcher searcher, SearchGroups cluster, + private static Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackend searcher, SearchGroups cluster, InvokerFactory invokerFactory, int maxHitsPerNode) { String searchPath = query.getModel().getSearchPath(); if (searchPath == null) return Optional.empty(); @@ -297,7 +297,7 @@ public class Dispatcher extends AbstractComponent { } } - private static SearchInvoker getInternalInvoker(Query query, VespaBackEndSearcher searcher, SearchCluster cluster, + private static SearchInvoker getInternalInvoker(Query query, VespaBackend searcher, SearchCluster cluster, LoadBalancer loadBalancer, InvokerFactory invokerFactory, int maxHitsPerNode) { Optional<Node> directNode = cluster.localCorpusDispatchTarget(); if (directNode.isPresent()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f1d58122570..d7fad148c8c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -2,7 +2,7 @@ package com.yahoo.search.dispatch; import com.yahoo.concurrent.Timer; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; @@ -34,12 +34,12 @@ public abstract class InvokerFactory { this.hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); } - protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node); - public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); + public abstract FillInvoker createFillInvoker(VespaBackend searcher, Result result); /** * Creates a {@link SearchInvoker} for a list of content nodes. @@ -52,7 +52,7 @@ public abstract class InvokerFactory { * @return the invoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ - Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, + Optional<SearchInvoker> createSearchInvoker(VespaBackend searcher, Query query, List<Node> nodes, boolean acceptIncompleteCoverage, diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 9da4c91cd16..284b4a4fcbc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -14,7 +14,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; @@ -227,7 +227,7 @@ public class ProtobufSerialization { return convertFromResult(searchResult).toByteArray(); } - static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) + static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackend searcher, int partId, int distKey) throws InvalidProtocolBufferException { var protobuf = SearchProtocol.SearchReply.parseFrom(payload); return convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 8221391afe6..9fd602032e9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.FillInvoker; @@ -37,12 +37,12 @@ public class RpcInvokerFactory extends InvokerFactory { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, int maxHits, Node node) { + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node) { return Optional.of(new RpcSearchInvoker(searcher, compressor, node, rpcResourcePool, maxHits)); } @Override - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 93e09a04180..fb028358caf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -2,14 +2,13 @@ package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.Compressor; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.Optional; @@ -26,7 +25,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private static final String RPC_METHOD = "vespa.searchprotocol.search"; - private final VespaBackEndSearcher searcher; + private final VespaBackend searcher; private final Node node; private final RpcConnectionPool resourcePool; private final BlockingQueue<Client.ResponseOrError<ProtobufResponse>> responses; @@ -35,7 +34,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private Query query; - RpcSearchInvoker(VespaBackEndSearcher searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { + RpcSearchInvoker(VespaBackend searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { super(Optional.of(node)); this.searcher = searcher; this.node = node; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0b3179f5ad6..6eb69c76afd 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -534,7 +534,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { */ @Deprecated // TODO Remove on Vespa 9. public void fillAttributes(Result result) { - fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 0685a77d7a9..64a989fd0e7 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; import com.yahoo.component.chain.dependencies.After; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index 5fef90c2012..e367bb58f80 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import static com.yahoo.vespa.streamingvisitors.StreamingSearcher.STREAMING_STATISTICS; +import static com.yahoo.vespa.streamingvisitors.StreamingBackend.STREAMING_STATISTICS; /** * Generates mail-specific query metrics. diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java index 2f3b48b5683..9953d76f50a 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java @@ -14,7 +14,7 @@ import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.TimeoutException; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -44,7 +44,7 @@ import java.util.logging.Logger; * @author baldersheim * @author Ulf Carlin */ -public class StreamingSearcher extends VespaBackEndSearcher { +public class StreamingBackend extends VespaBackend { private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); @@ -53,7 +53,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; private final TracingOptions tracingOptions; - private static final Logger log = Logger.getLogger(StreamingSearcher.class.getName()); + private static final Logger log = Logger.getLogger(StreamingBackend.class.getName()); private Route route; @@ -63,17 +63,17 @@ public class StreamingSearcher extends VespaBackEndSearcher { /** The route to the storage cluster. */ private String storageClusterRouteSpec = null; - StreamingSearcher(VisitorFactory visitorFactory) { + StreamingBackend(VisitorFactory visitorFactory) { this.visitorFactory = visitorFactory; tracingOptions = TracingOptions.DEFAULT; } - StreamingSearcher(VisitorFactory visitorFactory, TracingOptions tracingOptions) { + StreamingBackend(VisitorFactory visitorFactory, TracingOptions tracingOptions) { this.visitorFactory = visitorFactory; this.tracingOptions = tracingOptions; } - public StreamingSearcher(VespaDocumentAccess access) { + public StreamingBackend(VespaDocumentAccess access) { this(new VespaVisitorFactory(access)); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java index e638e0cf12a..bb155e3f934 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java @@ -267,7 +267,7 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { log.log(Level.FINE, () -> "StreamingVisitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); - throw new TimeoutException("Query timed out in " + StreamingSearcher.class.getName()); + throw new TimeoutException("Query timed out in " + StreamingBackend.class.getName()); } } finally { session.destroy(); diff --git a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def index 3ef3e9ef824..71c88b79ecd 100644 --- a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def @@ -6,24 +6,4 @@ namespace=prelude.fastsearch ## The name of the schema/document database documentdb[].name string - -## The id of the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].id int -## The name of the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].name string -## The name of a field in the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].name string -## The type of a field in the summary class. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].type string -## Whether this field is a dynamic snippet. Not used TODO: Remove after July 2022 -documentdb[].summaryclass[].fields[].dynamic bool default=false - -## Information about rank profiles. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].name string -documentdb[].rankprofile[].hasSummaryFeatures bool default=true -documentdb[].rankprofile[].hasRankFeatures bool default=true - -# The name of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].input[].name string -# The tensor type of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 -documentdb[].rankprofile[].input[].type string +documentdb[].mode enum {INDEX, STREAMING, STORE_ONLY} default=INDEX diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index affa6161440..ff91f346195 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -9,7 +9,7 @@ import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -38,7 +39,6 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -51,21 +51,6 @@ public class ClusterSearcherTestCase { private static final double DELTA = 0.0000000000000001; - @Test - void testNoBackends() { - ClusterSearcher cluster = new ClusterSearcher(createSchemaInfo(), Set.of("dummy")); - try { - Execution execution = new Execution(cluster, Execution.Context.createContextStub()); - Query query = new Query("query=hello"); - query.setHits(10); - com.yahoo.search.Result result = execution.search(query); - assertNotNull(result.hits().getError()); - assertEquals("No backends in service. Try later", result.hits().getError().getMessage()); - } finally { - cluster.deconstruct(); - } - } - private static SchemaInfo createSchemaInfo() { var schemas = Stream.of("type1", "type2", "type3", "type4", "type5", "type6") .map(name -> new Schema.Builder(name).build()).toList(); @@ -89,10 +74,11 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolved() { + var backend = new MyMockBackend(false); SchemaInfo schemaInfo = createSchemaInfo(); - ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { - ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Map.of("type6", backend)); try { assertEquals(Set.of("type1", "type2", "type3"), resolve(cluster1, "")); assertEquals(Set.of("type6"), resolve(type1, "")); @@ -129,10 +115,11 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolvedTODO_REMOVE() { + var backend = new MyMockBackend(false); SchemaInfo schemaInfo = createSchemaInfo(); - ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backend, "type2", backend, "type3", backend)); try { - ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Map.of("type6", backend)); try { assertEquals(Set.of(), resolve(cluster1, "&sources=cluster2")); } finally { @@ -143,13 +130,38 @@ public class ClusterSearcherTestCase { } } - private static class MyMockSearcher extends VespaBackEndSearcher { + @Test + void testThatMultipleBackendsAreUsed() { + var backendA = new MyMockBackend(false); + var backendB = new MyMockBackend(false); + SchemaInfo schemaInfo = createSchemaInfo(); + var cluster1 = new ClusterSearcher(schemaInfo, Map.of("type1", backendA, "type2", backendB, "type3", backendA), + new InThreadExecutorService()); + try { + Execution execution = new Execution(cluster1, Execution.Context.createContextStub()); + execution.search(new Query("?query=hello")); + assertEquals(2, backendA.queries().size()); + assertEquals(1, backendB.queries().size()); + execution.search(new Query("?query=hello&restrict=type1")); + assertEquals(3, backendA.queries().size()); + assertEquals(1, backendB.queries().size()); + execution.search(new Query("?query=hello&restrict=type2,type3")); + assertEquals(4, backendA.queries().size()); + assertEquals(2, backendB.queries().size()); + } finally { + cluster1.deconstruct(); + } + } + + private static class MyMockBackend extends VespaBackend { private final String type1 = "type1"; private final String type2 = "type2"; private final String type3 = "type3"; private final Map<String, List<Hit>> results = new LinkedHashMap<>(); private final boolean expectAttributePrefetch; + + private final List<Query> queries = new ArrayList<>(); static final String ATTRIBUTE_PREFETCH = "attributeprefetch"; private String getId(String type, int i) { @@ -200,7 +212,7 @@ public class ClusterSearcherTestCase { createHit(getId(type3, 2), 5))); } - MyMockSearcher(boolean expectAttributePrefetch) { + MyMockBackend(boolean expectAttributePrefetch) { this.expectAttributePrefetch = expectAttributePrefetch; init(); } @@ -210,8 +222,11 @@ public class ClusterSearcherTestCase { return null; // search() is overriden, this should never be called } + List<Query> queries() { return queries; } + @Override public com.yahoo.search.Result search(String schema, Query query) { + queries.add(query); com.yahoo.search.Result result = new com.yahoo.search.Result(query); List<Hit> hits = getHits(query); if (hits != null) { @@ -267,10 +282,14 @@ public class ClusterSearcherTestCase { } private Execution createExecution(List<String> docTypesList, boolean expectAttributePrefetch) { + var backend = new MyMockBackend(expectAttributePrefetch); + Map<String, VespaBackend> searchers = new HashMap<>(); + for(String schema : docTypesList) { + searchers.put(schema, backend); + } Set<String> documentTypes = new LinkedHashSet<>(docTypesList); ClusterSearcher cluster = new ClusterSearcher(toSchemaInfo(documentTypes, "mycluster"), - documentTypes, - new MyMockSearcher(expectAttributePrefetch), + searchers, new InThreadExecutorService()); try { List<Schema> schemas = new ArrayList<>(); @@ -428,7 +447,7 @@ public class ClusterSearcherTestCase { assertResult(6, List.of(7.0, 9.0), getResult(3, 2, extra, ex)); assertResult(6, List.of(9.0, 10.0), getResult(4, 2, extra, ex)); assertResult(6, List.of(10.0), getResult(5, 2, extra, ex)); - assertResult(6, List.of(), getResult(6, 2, extra, ex)); + assertResult(6, List.of(), getResult(6, 2, extra, ex)); } private static ClusterSearcher createSearcher(String clusterName, Double maxQueryTimeout, Double maxQueryCacheTimeout, @@ -449,7 +468,11 @@ public class ClusterSearcherTestCase { clusterConfig.maxQueryCacheTimeout(maxQueryCacheTimeout); DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); - documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); + documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder() + .name("type1") + .mode(streamingMode + ? DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING + : DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX)); var schema = new Schema.Builder("type1"); DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java index ade094115fe..ba9988b865c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumDefinitionTestCase.java @@ -1,14 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; +package com.yahoo.prelude.fastsearch; -import com.yahoo.prelude.fastsearch.ByteField; -import com.yahoo.prelude.fastsearch.DataField; -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.IntegerField; -import com.yahoo.prelude.fastsearch.StringField; -import com.yahoo.document.DocumentId; import com.yahoo.document.GlobalId; import com.yahoo.search.schema.DocumentSummary; import com.yahoo.search.schema.Schema; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java index 8270700a66b..917206bf00c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java @@ -1,14 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; +package com.yahoo.prelude.fastsearch; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; -import com.yahoo.prelude.fastsearch.ClusterParams; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; -import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.MockDispatcher; @@ -38,14 +33,14 @@ import static org.junit.jupiter.api.Assertions.*; * * @author bratseth */ -public class FastSearcherTestCase { +public class IndexedBackendTestCase { private static final String SCHEMA = "test"; private static final String CLUSTER = "test"; @Test void testNullQuery() { - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - FastSearcher fastSearcher = new FastSearcher("container.0", + Logger.getLogger(IndexedBackend.class.getName()).setLevel(Level.ALL); + IndexedBackend fastSearcher = new IndexedBackend("container.0", MockDispatcher.create(List.of()), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -62,7 +57,7 @@ public class FastSearcherTestCase { assertEquals(Error.NULL_QUERY.code, message.getCode()); } - private Result doSearch(VespaBackEndSearcher searcher, Query query, int offset, int hits) { + private Result doSearch(VespaBackend searcher, Query query, int offset, int hits) { query.setOffset(offset); query.setHits(hits); return searcher.search(SCHEMA, query); @@ -70,7 +65,7 @@ public class FastSearcherTestCase { @Test void testSinglePassGroupingIsForcedWithSingleNodeGroups() { - FastSearcher fastSearcher = new FastSearcher("container.0", + IndexedBackend fastSearcher = new IndexedBackend("container.0", MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -93,7 +88,7 @@ public class FastSearcherTestCase { @Test void testRankProfileValidation() { - FastSearcher fastSearcher = new FastSearcher("container.0", + IndexedBackend fastSearcher = new IndexedBackend("container.0", MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -112,7 +107,7 @@ public class FastSearcherTestCase { .add(new RankProfile.Builder("default").setHasRankFeatures(false) .setHasSummaryFeatures(false) .build()); - FastSearcher backend = new FastSearcher("container.0", + IndexedBackend backend = new IndexedBackend("container.0", MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -132,7 +127,7 @@ public class FastSearcherTestCase { void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0), new Node(CLUSTER, 2, "host1", 0))); - FastSearcher fastSearcher = new FastSearcher("container.0", + IndexedBackend fastSearcher = new IndexedBackend("container.0", dispatcher, new SummaryParameters(null), new ClusterParams("testhittype"), @@ -182,11 +177,11 @@ public class FastSearcherTestCase { assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus } - private String searchError(String query, VespaBackEndSearcher searcher) { + private String searchError(String query, VespaBackend searcher) { return search(query, searcher).hits().getError().getDetailedMessage(); } - private Result search(String query, VespaBackEndSearcher searcher) { + private Result search(String query, VespaBackend searcher) { return searcher.search(SCHEMA, new Query(query)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java index a033ccdace6..7760e204d4b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/PartialFillTestCase.java @@ -1,12 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; +package com.yahoo.prelude.fastsearch; -import com.yahoo.component.chain.Chain; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.Searcher; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import org.junit.jupiter.api.Test; @@ -22,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; */ public class PartialFillTestCase { - public static class FS4 extends VespaBackEndSearcher { + public static class FS4 extends VespaBackend { public List<Result> history = new ArrayList<>(); protected Result doSearch2(String schema, Query query) { return new Result(query); @@ -32,7 +28,7 @@ public class PartialFillTestCase { } } - public static class BadFS4 extends VespaBackEndSearcher { + public static class BadFS4 extends VespaBackend { protected Result doSearch2(String schema, Query query) { return new Result(query); } @@ -137,14 +133,8 @@ public class PartialFillTestCase { } } - private void doFill(VespaBackEndSearcher searcher, Result result, String summaryClass) { + private void doFill(VespaBackend searcher, Result result, String summaryClass) { searcher.fill(result, summaryClass); } - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java deleted file mode 100644 index 0a3e7bda318..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.test; - -import com.yahoo.search.result.Hit; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.jupiter.api.Test; - -/** - * Runs a query thru the configured search chain from a real http channel - * to a mock fdispatch channel. The setup is rather complicated, as the point is - * to span as much of the processing from query to result as possible. - * - * @author bratseth - */ -public class IntegrationTestCase { - - public static class SecondSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result = execution.search(query); - result.hits().add(new Hit("searcher:2",1000)); - return result; - } - } - public static class ThirdSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result = execution.search(query); - result.hits().add(new Hit("searcher:3",1000)); - return result; - } - } - - @Test - void testQuery() throws java.io.IOException { - /* - TODO: (JSB) This blocks forever on Linux (not Windows) because - the ServerSocketChannel.accept method in Server - seems to starve the test running thread, - causing it to get stuck in waitForServerInitialization. - This must be caused by starvation because - replacing the test with Thread.sleep(n) - gives the same result if n is large enough (2000 - is large enough, 1000 is not. - Resolve this in some way, perhaps by switching to - non-blocking io (and then remember to remove the next line). - */ - } - - /* - if (1==1) return; - ServerThread serverThread=new ServerThread(); - try { - serverThread.start(); - waitForServerInitialization(); - insertMockFs4Channel(); - ByteBuffer buffer=ByteBuffer.allocate(4096); - buffer.put(getBytes("GET /?query=hans HTTP/1.1\n\n")); - SocketChannel socket= - SocketChannel.open(new InetSocketAddress(Server.get().getHost(), - Server.get().getPort())); - buffer.flip(); - socket.write(buffer); - - buffer.clear(); - socket.read(buffer); - // TODO: Validate return too - - } - finally { - serverThread.interrupt(); - } - } - - private static void assertCorrectQueryData(QueryPacket packet) { - assertEquals("Query x packet " + - "[query: query 'RANK hans bcatpat.bidxpatlvl1:hans' [path='/']]", - packet.toString()); - } - - private void insertMockFs4Channel() { - Searcher current=SearchChain.get(); - while (current.getChained().getChained()!=null) - current=current.getChained(); - assertTrue(current.getChained() instanceof FastSearcher); - FastSearcher mockFastSearcher= - new FastSearcher(new MockFSChannel(), - "file:etc/qr-summary.cf", - "testhittype"); - current.setChained(mockFastSearcher); - } - - private void waitForServerInitialization() { - int sleptMs=0; - while (Server.get().getHost()==null) { - try { Thread.sleep(10); } catch (Exception e) {} - sleptMs+=10; - } - } - - private class ServerThread extends Thread { - - public void run() { - try { - Server.get().start(8081,new SearchRequestHandler()); - } - catch (java.io.IOException e) { - throw new RuntimeException("Failed",e); - } - } - } - - private byte[] getBytes(String string) { - try { - return string.getBytes("utf-8"); - } - catch (java.io.UnsupportedEncodingException e) { - throw new RuntimeException("Won't happen",e); - } - } - */ - /** A channel which returns hardcoded packets of the same type as fdispatch */ - /* - private static class MockFSChannel extends Channel { - - public MockFSChannel() {} - - public void sendPacket(Packet packet) { - if (packet instanceof QueryPacket) { - assertCorrectQueryData((QueryPacket)packet); - } - else { - throw new RuntimeException("Mock channel don't know what to reply to " + - packet); - } - } - - public Packet[] receivePackets() { - List packets=new java.util.ArrayList(); - QueryResultPacket result=QueryResultPacket.create(); - result.addDocument(new DocumentInfo(123,2003,234,1000,1)); - result.addDocument(new DocumentInfo(456,1855,234,1001,1)); - packets.add(result); - addDocsums(packets); - return (Packet[])packets.toArray(new Packet[packets.size()]); - } - - private void addDocsums(List packets) { - ByteBuffer buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); - buffer.position(0); - packets.add(PacketDecoder.decode(buffer)); - - buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); - buffer.position(0); - packets.add(PacketDecoder.decode(buffer)); - - packets.add(EolPacket.create()); - } - - private ByteBuffer createDocsumPacketData(byte[] docsumData) { - ByteBuffer buffer=ByteBuffer.allocate(docsumData.length+12+4); - buffer.limit(buffer.capacity()); - buffer.position(0); - buffer.putInt(docsumData.length+8+4); - buffer.putInt(205); // Docsum packet code - buffer.putInt(0); - buffer.putInt(0); - buffer.put(docsumData); - return buffer; - } - - } - */ -} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 23846db3f0b..c966fbc200d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -3,7 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.compress.CompressionType; import com.yahoo.prelude.Pong; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; @@ -236,7 +236,7 @@ public class DispatcherTest { // This factory just forwards search to the dummy RPC layer above, nothing more. InvokerFactoryFactory invokerFactories = (rpcConnectionPool, searchGroups, dispatchConfig) -> new InvokerFactory(searchGroups, dispatchConfig) { - @Override protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, int maxHits, Node node) { + @Override protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node) { return Optional.of(new SearchInvoker(Optional.of(node)) { @Override protected Object sendSearchRequest(Query query, Object context) { rpcPool.getConnection(node.key()).request(null, null, 0, null, null, 0); @@ -248,7 +248,7 @@ public class DispatcherTest { @Override protected void release() { } }); }; - @Override public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + @Override public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { return new FillInvoker() { @Override protected void getFillResults(Result result, String summaryClass) { fail(); } @Override protected void sendFillRequest(Result result, String summaryClass) { fail(); } @@ -369,7 +369,7 @@ public class DispatcherTest { } @Override - public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, + public Optional<SearchInvoker> createSearchInvoker(VespaBackend searcher, Query query, List<Node> nodes, boolean acceptIncompleteCoverage, @@ -391,7 +391,7 @@ public class DispatcherTest { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHitsPerNode, Node node) { @@ -400,7 +400,7 @@ public class DispatcherTest { } @Override - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { fail("Unexpected call to createFillInvoker"); return null; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index 2600d8612f8..ef8e0522337 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -5,7 +5,7 @@ package com.yahoo.search.dispatch.rpc; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.CompressionType; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Node; @@ -117,8 +117,8 @@ public class RpcSearchInvokerTest { }; } - private VespaBackEndSearcher mockSearcher() { - return new VespaBackEndSearcher() { + private VespaBackend mockSearcher() { + return new VespaBackend() { @Override protected Result doSearch2(String schema, Query query) { fail("Unexpected call"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 4455c8a04a5..87d18c18db5 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.*; import java.util.Arrays; import java.util.List; -import java.util.Map; import com.yahoo.search.schema.DocumentSummary; import com.yahoo.search.schema.Schema; @@ -21,7 +20,7 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; /** diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java index 831261bb91a..5cc0e6b060b 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java @@ -134,9 +134,9 @@ public class MetricsSearcherTestCase { private void assignContextProperties(Query query, String loadType) { if (loadType != null && loadType.equals(LOADTYPE1)) { - query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, visitorStats); + query.getContext(true).setProperty(StreamingBackend.STREAMING_STATISTICS, visitorStats); } else { - query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, null); + query.getContext(true).setProperty(StreamingBackend.STREAMING_STATISTICS, null); } } } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index c7e95f802ab..b96dd97f76c 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -159,7 +159,7 @@ public class StreamingSearcherTestCase { } } - private static Result executeQuery(StreamingSearcher searcher, Query query) { + private static Result executeQuery(StreamingBackend searcher, Query query) { return searcher.doSearch2("test", query); } @@ -181,7 +181,7 @@ public class StreamingSearcherTestCase { return queries; } - private static void checkError(StreamingSearcher searcher, String queryString, String message, String detailedMessage) { + private static void checkError(StreamingBackend searcher, String queryString, String message, String detailedMessage) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNotNull(result.hits().getError()); @@ -194,7 +194,7 @@ public class StreamingSearcherTestCase { } } - private static void checkSearch(StreamingSearcher searcher, String queryString, int hitCount, String idPrefix) { + private static void checkSearch(StreamingBackend searcher, String queryString, int hitCount, String idPrefix) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNull(result.hits().getError()); @@ -212,11 +212,11 @@ public class StreamingSearcherTestCase { } } - private static void checkGrouping(StreamingSearcher searcher, String queryString, int hitCount) { + private static void checkGrouping(StreamingBackend searcher, String queryString, int hitCount) { checkSearch(searcher, queryString, hitCount, null); } - private static void checkMatchFeatures(StreamingSearcher searcher) { + private static void checkMatchFeatures(StreamingBackend searcher) { String queryString = "/?streaming.selection=true&query=match_features"; Result result = executeQuery(searcher, new Query(queryString)); assertNull(result.hits().getError()); @@ -229,7 +229,7 @@ public class StreamingSearcherTestCase { @Test void testBasics() { MockVisitorFactory factory = new MockVisitorFactory(); - StreamingSearcher searcher = new StreamingSearcher(factory); + StreamingBackend searcher = new StreamingBackend(factory); var schema = new Schema.Builder("test"); schema.add(new com.yahoo.search.schema.DocumentSummary.Builder("default").build()); @@ -276,25 +276,25 @@ public class StreamingSearcherTestCase { String groupId2 = "id:namespace:mytype:g=group2:userspecific"; String badId = "unknowscheme:namespace:something"; - assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, true)); - - assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(userId2, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId1, generalQuery, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId2, generalQuery, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, generalQuery, false)); - - assertTrue(StreamingSearcher.verifyDocId(userId1, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(userId2, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId1, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId2, user1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, user1Query, false)); - - assertFalse(StreamingSearcher.verifyDocId(userId1, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(userId2, group1Query, false)); - assertTrue(StreamingSearcher.verifyDocId(groupId1, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(groupId2, group1Query, false)); - assertFalse(StreamingSearcher.verifyDocId(badId, group1Query, false)); + assertTrue(StreamingBackend.verifyDocId(userId1, generalQuery, true)); + + assertTrue(StreamingBackend.verifyDocId(userId1, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(userId2, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(groupId1, generalQuery, false)); + assertTrue(StreamingBackend.verifyDocId(groupId2, generalQuery, false)); + assertFalse(StreamingBackend.verifyDocId(badId, generalQuery, false)); + + assertTrue(StreamingBackend.verifyDocId(userId1, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(userId2, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId1, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId2, user1Query, false)); + assertFalse(StreamingBackend.verifyDocId(badId, user1Query, false)); + + assertFalse(StreamingBackend.verifyDocId(userId1, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(userId2, group1Query, false)); + assertTrue(StreamingBackend.verifyDocId(groupId1, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(groupId2, group1Query, false)); + assertFalse(StreamingBackend.verifyDocId(badId, group1Query, false)); } private static class TraceFixture { @@ -304,13 +304,13 @@ public class StreamingSearcherTestCase { TracingOptions options; MockVisitorFactory factory; - StreamingSearcher searcher; + StreamingBackend searcher; private TraceFixture(Long firstTimestamp, Long... additionalTimestamps) { clock = MockUtils.mockedClockReturning(firstTimestamp, additionalTimestamps); options = new TracingOptions(sampler, exporter, clock, 8, 2.0); factory = new MockVisitorFactory(); - searcher = new StreamingSearcher(factory, options); + searcher = new StreamingBackend(factory, options); } private TraceFixture() { diff --git a/document/abi-spec.json b/document/abi-spec.json index ca06e2547d7..ca4b4da3ca0 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -2862,7 +2862,8 @@ "methods" : [ "public abstract void write(com.yahoo.document.Document)", "public abstract void write(com.yahoo.document.DocumentId)", - "public abstract void write(com.yahoo.document.DocumentType)" + "public abstract void write(com.yahoo.document.DocumentType)", + "public abstract void write(com.yahoo.document.DocumentRemove)" ], "fields" : [ ] }, @@ -3104,6 +3105,7 @@ "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", "public void write(com.yahoo.document.DocumentId)", "public void write(com.yahoo.document.DocumentType)", + "public void write(com.yahoo.document.DocumentRemove)", "public void write(com.yahoo.document.annotation.Annotation)", "public void write(com.yahoo.document.annotation.SpanTree)", "public void write(com.yahoo.document.annotation.SpanNode)", diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 7b1042903ec..ed6bdc721a0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -347,4 +347,5 @@ public class JsonSerializationHelper { wrapIOException(() -> generator.writeFieldName(field.getName())); } } + } diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 7e82e830064..9cbadb65f10 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.StreamReadConstraints; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; @@ -263,6 +264,19 @@ public class JsonWriter implements DocumentWriter { // NOP, fetched from Document } + public void write(DocumentRemove documentRemove) { + try { + generator.writeStartObject(); + + serializeStringField(generator, new FieldBase("remove"), new StringFieldValue(documentRemove.getId().toString())); + + generator.writeEndObject(); + generator.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Utility method to easily serialize a single document. * diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java index c84140c9ea0..10483d8609f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java @@ -3,6 +3,7 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; /** @@ -17,4 +18,6 @@ public interface DocumentWriter extends FieldWriter { void write(DocumentType type); + void write(DocumentRemove documentRemove); + } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 17ab3890bcf..4cb836860be 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -8,6 +8,7 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; @@ -426,6 +427,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume putShort(null, (short) 0); // Used to hold the version. Is now always 0. } + public void write(DocumentRemove documentRemove) { + throw new UnsupportedOperationException("serializing remove not implemented"); + } + public void write(Annotation annotation) { buf.putInt(annotation.getType().getId()); //name hash diff --git a/messagebus/src/tests/configagent/CMakeLists.txt b/messagebus/src/tests/configagent/CMakeLists.txt index 9cc9eacbe1c..9b07222fc74 100644 --- a/messagebus/src/tests/configagent/CMakeLists.txt +++ b/messagebus/src/tests/configagent/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(messagebus_configagent_test_app TEST DEPENDS messagebus_messagebus-test messagebus + GTest::gtest ) vespa_add_test(NAME messagebus_configagent_test_app COMMAND messagebus_configagent_test_app) diff --git a/messagebus/src/tests/configagent/configagent.cpp b/messagebus/src/tests/configagent/configagent.cpp index f93bfd6c841..e8cab1e679a 100644 --- a/messagebus/src/tests/configagent/configagent.cpp +++ b/messagebus/src/tests/configagent/configagent.cpp @@ -1,109 +1,113 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/messagebus/configagent.h> #include <vespa/messagebus/iconfighandler.h> #include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/config-messagebus.h> #include <vespa/config/print/fileconfigreader.hpp> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> using namespace mbus; using namespace messagebus; using namespace config; -class Test : public vespalib::TestApp, public IConfigHandler { -private: +class ConfigAgentTest : public testing::Test, public IConfigHandler { +protected: RoutingSpec _spec; + ConfigAgentTest(); + ~ConfigAgentTest() override; bool checkHalf(); bool checkFull(); - bool checkTables(uint32_t numTables); - -public: - ~Test() override; - int Main() override; + void checkTables(uint32_t numTables, bool& success); bool setupRouting(RoutingSpec spec) override; }; -Test::~Test() = default; +ConfigAgentTest::ConfigAgentTest() + : testing::Test(), + IConfigHandler(), + _spec() +{ +} -TEST_APPHOOK(Test); +ConfigAgentTest::~ConfigAgentTest() = default; bool -Test::setupRouting(RoutingSpec spec) +ConfigAgentTest::setupRouting(RoutingSpec spec) { _spec = std::move(spec); return true; } -bool -Test::checkTables(uint32_t numTables) +void +ConfigAgentTest::checkTables(uint32_t numTables, bool& success) { - if (!EXPECT_EQUAL(numTables, _spec.getNumTables())) return false; + ASSERT_EQ(numTables, _spec.getNumTables()); if (numTables > 0) { - if (!EXPECT_EQUAL("foo", _spec.getTable(0).getProtocol())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getNumHops())) return false; - if (!EXPECT_EQUAL("foo-h1", _spec.getTable(0).getHop(0).getName())) return false; - if (!EXPECT_EQUAL("foo-h1-sel", _spec.getTable(0).getHop(0).getSelector())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getHop(0).getNumRecipients())) return false; - if (!EXPECT_EQUAL("foo-h1-r1", _spec.getTable(0).getHop(0).getRecipient(0))) return false; - if (!EXPECT_EQUAL("foo-h1-r2", _spec.getTable(0).getHop(0).getRecipient(1))) return false; - if (!EXPECT_EQUAL(true, _spec.getTable(0).getHop(0).getIgnoreResult())) return false; - if (!EXPECT_EQUAL("foo-h2", _spec.getTable(0).getHop(1).getName())) return false; - if (!EXPECT_EQUAL("foo-h2-sel", _spec.getTable(0).getHop(1).getSelector())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getHop(1).getNumRecipients())) return false; - if (!EXPECT_EQUAL("foo-h2-r1", _spec.getTable(0).getHop(1).getRecipient(0))) return false; - if (!EXPECT_EQUAL("foo-h2-r2", _spec.getTable(0).getHop(1).getRecipient(1))) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getNumRoutes())) return false; - if (!EXPECT_EQUAL("foo-r1", _spec.getTable(0).getRoute(0).getName())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getRoute(0).getNumHops())) return false; - if (!EXPECT_EQUAL("foo-h1", _spec.getTable(0).getRoute(0).getHop(0))) return false; - if (!EXPECT_EQUAL("foo-h2", _spec.getTable(0).getRoute(0).getHop(1))) return false; - if (!EXPECT_EQUAL("foo-r2", _spec.getTable(0).getRoute(1).getName())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(0).getRoute(1).getNumHops())) return false; - if (!EXPECT_EQUAL("foo-h2", _spec.getTable(0).getRoute(1).getHop(0))) return false; - if (!EXPECT_EQUAL("foo-h1", _spec.getTable(0).getRoute(1).getHop(1))) return false; + ASSERT_EQ("foo", _spec.getTable(0).getProtocol()); + ASSERT_EQ(2u, _spec.getTable(0).getNumHops()); + ASSERT_EQ("foo-h1", _spec.getTable(0).getHop(0).getName()); + ASSERT_EQ("foo-h1-sel", _spec.getTable(0).getHop(0).getSelector()); + ASSERT_EQ(2u, _spec.getTable(0).getHop(0).getNumRecipients()); + ASSERT_EQ("foo-h1-r1", _spec.getTable(0).getHop(0).getRecipient(0)); + ASSERT_EQ("foo-h1-r2", _spec.getTable(0).getHop(0).getRecipient(1)); + ASSERT_EQ(true, _spec.getTable(0).getHop(0).getIgnoreResult()); + ASSERT_EQ("foo-h2", _spec.getTable(0).getHop(1).getName()); + ASSERT_EQ("foo-h2-sel", _spec.getTable(0).getHop(1).getSelector()); + ASSERT_EQ(2u, _spec.getTable(0).getHop(1).getNumRecipients()); + ASSERT_EQ("foo-h2-r1", _spec.getTable(0).getHop(1).getRecipient(0)); + ASSERT_EQ("foo-h2-r2", _spec.getTable(0).getHop(1).getRecipient(1)); + ASSERT_EQ(2u, _spec.getTable(0).getNumRoutes()); + ASSERT_EQ("foo-r1", _spec.getTable(0).getRoute(0).getName()); + ASSERT_EQ(2u, _spec.getTable(0).getRoute(0).getNumHops()); + ASSERT_EQ("foo-h1", _spec.getTable(0).getRoute(0).getHop(0)); + ASSERT_EQ("foo-h2", _spec.getTable(0).getRoute(0).getHop(1)); + ASSERT_EQ("foo-r2", _spec.getTable(0).getRoute(1).getName()); + ASSERT_EQ(2u, _spec.getTable(0).getRoute(1).getNumHops()); + ASSERT_EQ("foo-h2", _spec.getTable(0).getRoute(1).getHop(0)); + ASSERT_EQ("foo-h1", _spec.getTable(0).getRoute(1).getHop(1)); } if (numTables > 1) { - if (!EXPECT_EQUAL("bar", _spec.getTable(1).getProtocol())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getNumHops())) return false; - if (!EXPECT_EQUAL("bar-h1", _spec.getTable(1).getHop(0).getName())) return false; - if (!EXPECT_EQUAL("bar-h1-sel", _spec.getTable(1).getHop(0).getSelector())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getHop(0).getNumRecipients())) return false; - if (!EXPECT_EQUAL("bar-h1-r1", _spec.getTable(1).getHop(0).getRecipient(0))) return false; - if (!EXPECT_EQUAL("bar-h1-r2", _spec.getTable(1).getHop(0).getRecipient(1))) return false; - if (!EXPECT_EQUAL("bar-h2", _spec.getTable(1).getHop(1).getName())) return false; - if (!EXPECT_EQUAL("bar-h2-sel", _spec.getTable(1).getHop(1).getSelector())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getHop(1).getNumRecipients())) return false; - if (!EXPECT_EQUAL("bar-h2-r1", _spec.getTable(1).getHop(1).getRecipient(0))) return false; - if (!EXPECT_EQUAL("bar-h2-r2", _spec.getTable(1).getHop(1).getRecipient(1))) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getNumRoutes())) return false; - if (!EXPECT_EQUAL("bar-r1", _spec.getTable(1).getRoute(0).getName())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getRoute(0).getNumHops())) return false; - if (!EXPECT_EQUAL("bar-h1", _spec.getTable(1).getRoute(0).getHop(0))) return false; - if (!EXPECT_EQUAL("bar-h2", _spec.getTable(1).getRoute(0).getHop(1))) return false; - if (!EXPECT_EQUAL("bar-r2", _spec.getTable(1).getRoute(1).getName())) return false; - if (!EXPECT_EQUAL(2u, _spec.getTable(1).getRoute(1).getNumHops())) return false; - if (!EXPECT_EQUAL("bar-h2", _spec.getTable(1).getRoute(1).getHop(0))) return false; - if (!EXPECT_EQUAL("bar-h1", _spec.getTable(1).getRoute(1).getHop(1))) return false; + ASSERT_EQ("bar", _spec.getTable(1).getProtocol()); + ASSERT_EQ(2u, _spec.getTable(1).getNumHops()); + ASSERT_EQ("bar-h1", _spec.getTable(1).getHop(0).getName()); + ASSERT_EQ("bar-h1-sel", _spec.getTable(1).getHop(0).getSelector()); + ASSERT_EQ(2u, _spec.getTable(1).getHop(0).getNumRecipients()); + ASSERT_EQ("bar-h1-r1", _spec.getTable(1).getHop(0).getRecipient(0)); + ASSERT_EQ("bar-h1-r2", _spec.getTable(1).getHop(0).getRecipient(1)); + ASSERT_EQ("bar-h2", _spec.getTable(1).getHop(1).getName()); + ASSERT_EQ("bar-h2-sel", _spec.getTable(1).getHop(1).getSelector()); + ASSERT_EQ(2u, _spec.getTable(1).getHop(1).getNumRecipients()); + ASSERT_EQ("bar-h2-r1", _spec.getTable(1).getHop(1).getRecipient(0)); + ASSERT_EQ("bar-h2-r2", _spec.getTable(1).getHop(1).getRecipient(1)); + ASSERT_EQ(2u, _spec.getTable(1).getNumRoutes()); + ASSERT_EQ("bar-r1", _spec.getTable(1).getRoute(0).getName()); + ASSERT_EQ(2u, _spec.getTable(1).getRoute(0).getNumHops()); + ASSERT_EQ("bar-h1", _spec.getTable(1).getRoute(0).getHop(0)); + ASSERT_EQ("bar-h2", _spec.getTable(1).getRoute(0).getHop(1)); + ASSERT_EQ("bar-r2", _spec.getTable(1).getRoute(1).getName()); + ASSERT_EQ(2u, _spec.getTable(1).getRoute(1).getNumHops()); + ASSERT_EQ("bar-h2", _spec.getTable(1).getRoute(1).getHop(0)); + ASSERT_EQ("bar-h1", _spec.getTable(1).getRoute(1).getHop(1)); } - return true; + success = true; } bool -Test::checkHalf() +ConfigAgentTest::checkHalf() { - return _spec.getNumTables() == 1 && EXPECT_TRUE(checkTables(1)); + bool success = false; + return _spec.getNumTables() == 1 && (checkTables(1, success), success); } bool -Test::checkFull() +ConfigAgentTest::checkFull() { - return _spec.getNumTables() == 2 && EXPECT_TRUE(checkTables(2)); + bool success = false; + return _spec.getNumTables() == 2 && (checkTables(2, success), success); } -int -Test::Main() +TEST_F(ConfigAgentTest, test_config_agent) { - TEST_INIT("configagent_test"); EXPECT_TRUE(!checkHalf()); EXPECT_TRUE(!checkFull()); ConfigAgent agent(*this); @@ -118,5 +122,6 @@ Test::Main() agent.configure(FileConfigReader<MessagebusConfig>(TEST_PATH("full.cfg")).read()); EXPECT_TRUE(checkFull()); EXPECT_TRUE(!checkHalf()); - TEST_DONE(); } + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/messagebus/src/tests/messagebus/CMakeLists.txt b/messagebus/src/tests/messagebus/CMakeLists.txt index 3f79e2ed851..f05141ca28f 100644 --- a/messagebus/src/tests/messagebus/CMakeLists.txt +++ b/messagebus/src/tests/messagebus/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(messagebus_messagebus_test_app TEST DEPENDS messagebus_messagebus-test messagebus + GTest::gtest ) vespa_add_test(NAME messagebus_messagebus_test_app COMMAND messagebus_messagebus_test_app) diff --git a/messagebus/src/tests/messagebus/messagebus.cpp b/messagebus/src/tests/messagebus/messagebus.cpp index 5c1bdc54f00..e4d90a3c198 100644 --- a/messagebus/src/tests/messagebus/messagebus.cpp +++ b/messagebus/src/tests/messagebus/messagebus.cpp @@ -11,8 +11,8 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> #include <thread> using namespace mbus; @@ -98,8 +98,8 @@ struct Search : public Server { //----------------------------------------------------------------------------- -class Test : public vespalib::TestApp { -private: +class MessageBusTest : public testing::Test { +protected: Slobrok::UP slobrok; Client::UP client; DocProc::UP dp0; @@ -112,44 +112,21 @@ private: std::vector<DocProc*> dpVec; std::vector<Search*> searchVec; -public: - Test(); - ~Test(); - int Main() override; - void testSendToCol(); - void testDirectHop(); - void testDirectRoute(); - void testRoutingPolicyCache(); - -private: - void setup(); - void teardown(); + MessageBusTest(); + ~MessageBusTest(); + void SetUp() override; + void TearDown() override; void assertSrc(Client& src); void assertItr(DocProc& itr); void assertDst(Search& dst); }; -TEST_APPHOOK(Test); - -Test::Test() = default; -Test::~Test() = default; - -int -Test::Main() -{ - TEST_INIT("messagebus_test"); - - testSendToCol(); TEST_FLUSH(); - testDirectHop(); TEST_FLUSH(); - testDirectRoute(); TEST_FLUSH(); - testRoutingPolicyCache(); TEST_FLUSH(); - - TEST_DONE(); -} +MessageBusTest::MessageBusTest() = default; +MessageBusTest::~MessageBusTest() = default; void -Test::setup() +MessageBusTest::SetUp() { slobrok.reset(new Slobrok()); client.reset(new Client(*slobrok)); @@ -188,7 +165,8 @@ Test::setup() ASSERT_TRUE(dp2->server.waitSlobrok("search/r.1/c.1/session")); } -void Test::teardown() +void +MessageBusTest::TearDown() { dpVec.clear(); searchVec.clear(); @@ -203,10 +181,8 @@ void Test::teardown() slobrok.reset(); } -void -Test::testSendToCol() +TEST_F(MessageBusTest, test_send_to_col) { - setup(); ASSERT_TRUE(SimpleMessage("msg").getHash() % 2 == 0); for (uint32_t i = 0; i < 150; ++i) { Message::UP msg(new SimpleMessage("msg")); @@ -244,13 +220,10 @@ Test::testSendToCol() ASSERT_TRUE(reply->isReply()); EXPECT_TRUE(static_cast<Reply&>(*reply).getNumErrors() == 0); } - teardown(); } -void -Test::testDirectHop() +TEST_F(MessageBusTest, test_direct_hop) { - setup(); for (int row = 0; row < 2; row++) { for (int col = 0; col < 2; col++) { Search* dst = searchVec[row * 2 + col]; @@ -260,25 +233,22 @@ Test::testDirectHop() Message::UP(new SimpleMessage("empty")), Route().addHop(vespalib::make_string("search/r.%d/c.%d/session", row, col))) .isAccepted()); - assertDst(*dst); - assertSrc(*client); + ASSERT_NO_FATAL_FAILURE(assertDst(*dst)); + ASSERT_NO_FATAL_FAILURE(assertSrc(*client)); // Send using address. ASSERT_TRUE(client->session->send( Message::UP(new SimpleMessage("empty")), Route().addHop(Hop(dst->session->getConnectionSpec().c_str()))) .isAccepted()); - assertDst(*dst); - assertSrc(*client); + ASSERT_NO_FATAL_FAILURE(assertDst(*dst)); + ASSERT_NO_FATAL_FAILURE(assertSrc(*client)); } } - teardown(); } -void -Test::testDirectRoute() +TEST_F(MessageBusTest, test_direct_route) { - setup(); ASSERT_TRUE(client->session->send( Message::UP(new SimpleMessage("empty")), Route() @@ -290,26 +260,24 @@ Test::testDirectRoute() .addHop(Hop(dp2->session->getConnectionSpec())) .addHop(Hop("search/r.0/c.0/session"))) .isAccepted()); - assertItr(*dp0); - assertItr(*dp0); - assertItr(*dp1); - assertItr(*dp1); - assertItr(*dp2); - assertItr(*dp2); - assertDst(*search00); - assertItr(*dp2); - assertItr(*dp2); - assertItr(*dp1); - assertItr(*dp1); - assertItr(*dp0); - assertItr(*dp0); - assertSrc(*client); - - teardown(); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp0)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp0)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp1)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp1)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp2)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp2)); + ASSERT_NO_FATAL_FAILURE(assertDst(*search00)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp2)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp2)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp1)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp1)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp0)); + ASSERT_NO_FATAL_FAILURE(assertItr(*dp0)); + ASSERT_NO_FATAL_FAILURE(assertSrc(*client)); } void -Test::assertDst(Search& dst) +MessageBusTest::assertDst(Search& dst) { ASSERT_TRUE(dst.waitQueueSize(1)); Routable::UP msg = dst.queue.dequeue(); @@ -318,7 +286,7 @@ Test::assertDst(Search& dst) } void -Test::assertItr(DocProc& itr) +MessageBusTest::assertItr(DocProc& itr) { ASSERT_TRUE(itr.waitQueueSize(1)); Routable::UP msg = itr.queue.dequeue(); @@ -327,17 +295,15 @@ Test::assertItr(DocProc& itr) } void -Test::assertSrc(Client& src) +MessageBusTest::assertSrc(Client& src) { ASSERT_TRUE(src.waitQueueSize(1)); Routable::UP msg = src.queue.dequeue(); ASSERT_TRUE(msg); } -void -Test::testRoutingPolicyCache() +TEST_F(MessageBusTest, test_routing_policy_cache) { - setup(); MessageBus &bus = client->server.mb; IRoutingPolicy::SP all = bus.getRoutingPolicy(SimpleProtocol::NAME, "All", ""); @@ -354,6 +320,6 @@ Test::testRoutingPolicyCache() IRoutingPolicy::SP refArg = bus.getRoutingPolicy(SimpleProtocol::NAME, "All", "Arg"); ASSERT_TRUE(refArg.get() != NULL); ASSERT_TRUE(allArg.get() == refArg.get()); - - teardown(); } + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/messagebus/src/tests/resender/CMakeLists.txt b/messagebus/src/tests/resender/CMakeLists.txt index 2d0e5dbbb61..c364806b2dd 100644 --- a/messagebus/src/tests/resender/CMakeLists.txt +++ b/messagebus/src/tests/resender/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(messagebus_resender_test_app TEST DEPENDS messagebus_messagebus-test messagebus + GTest::gtest ) vespa_add_test(NAME messagebus_resender_test_app COMMAND messagebus_resender_test_app) diff --git a/messagebus/src/tests/resender/resender.cpp b/messagebus/src/tests/resender/resender.cpp index b9c75c36dd0..e61d3704909 100644 --- a/messagebus/src/tests/resender/resender.cpp +++ b/messagebus/src/tests/resender/resender.cpp @@ -9,26 +9,11 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace mbus; -//////////////////////////////////////////////////////////////////////////////// -// -// Utilities -// -//////////////////////////////////////////////////////////////////////////////// - -class StringList : public std::vector<string> { -public: - StringList &add(const string &str); -}; - -StringList & -StringList::add(const string &str) -{ - std::vector<string>::push_back(str); return *this; -} +namespace { static const duration GET_MESSAGE_TIMEOUT = 60s; @@ -55,35 +40,17 @@ public: bool start(); }; -class Test : public vespalib::TestApp { -private: - Message::UP createMessage(const string &msg); - void replyFromDestination(TestData &data, Message::UP msg, uint32_t errorCode, double retryDelay); - -public: - int Main() override; - void testRetryTag(TestData &data); - void testRetryEnabledTag(TestData &data); - void testTransientError(TestData &data); - void testFatalError(TestData &data); - void testDisableRetry(TestData &data); - void testRetryDelay(TestData &data); - void testRequestRetryDelay(TestData &data); -}; - -TEST_APPHOOK(Test); - -TestData::TestData() : - _slobrok(), - _retryPolicy(new RetryTransientErrorsPolicy()), - _srcServer(MessageBusParams().setRetryPolicy(_retryPolicy).addProtocol(std::make_shared<SimpleProtocol>()), - RPCNetworkParams(_slobrok.config())), - _srcSession(), - _srcHandler(), - _dstServer(MessageBusParams().addProtocol(std::make_shared<SimpleProtocol>()), - RPCNetworkParams(_slobrok.config()).setIdentity(Identity("dst"))), - _dstSession(), - _dstHandler() +TestData::TestData() + : _slobrok(), + _retryPolicy(new RetryTransientErrorsPolicy()), + _srcServer(MessageBusParams().setRetryPolicy(_retryPolicy).addProtocol(std::make_shared<SimpleProtocol>()), + RPCNetworkParams(_slobrok.config())), + _srcSession(), + _srcHandler(), + _dstServer(MessageBusParams().addProtocol(std::make_shared<SimpleProtocol>()), + RPCNetworkParams(_slobrok.config()).setIdentity(Identity("dst"))), + _dstSession(), + _dstHandler() { } TestData::~TestData() = default; @@ -105,35 +72,16 @@ TestData::start() return true; } -Message::UP -Test::createMessage(const string &msg) +std::unique_ptr<Message> +createMessage(const string &msg) { Message::UP ret(new SimpleMessage(msg)); ret->getTrace().setLevel(9); return ret; } -int -Test::Main() -{ - TEST_INIT("resender_test"); - - TestData data; - ASSERT_TRUE(data.start()); - - testRetryTag(data); TEST_FLUSH(); - testRetryEnabledTag(data); TEST_FLUSH(); - testTransientError(data); TEST_FLUSH(); - testFatalError(data); TEST_FLUSH(); - testDisableRetry(data); TEST_FLUSH(); - testRetryDelay(data); TEST_FLUSH(); - testRequestRetryDelay(data); TEST_FLUSH(); - - TEST_DONE(); -} - void -Test::replyFromDestination(TestData &data, Message::UP msg, uint32_t errorCode, double retryDelay) +replyFromDestination(TestData &data, Message::UP msg, uint32_t errorCode, double retryDelay) { Reply::UP reply(new EmptyReply()); reply->swapState(*msg); @@ -144,22 +92,51 @@ Test::replyFromDestination(TestData &data, Message::UP msg, uint32_t errorCode, data._dstSession->reply(std::move(reply)); } +} + +class ResenderTest : public testing::Test { +protected: + static std::shared_ptr<TestData> _data; + ResenderTest(); + ~ResenderTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +std::shared_ptr<TestData> ResenderTest::_data; + +ResenderTest::ResenderTest() = default; +ResenderTest::~ResenderTest() = default; + +void +ResenderTest::SetUpTestSuite() +{ + _data = std::make_shared<TestData>(); + ASSERT_TRUE(_data->start()); +} + +void +ResenderTest::TearDownTestSuite() +{ + _data.reset(); +} + //////////////////////////////////////////////////////////////////////////////// // // Tests // //////////////////////////////////////////////////////////////////////////////// -void -Test::testRetryTag(TestData &data) +TEST_F(ResenderTest, test_retry_tag) { + auto& data = *_data; data._retryPolicy->setEnabled(true); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); for (uint32_t i = 0; i < 5; ++i) { - EXPECT_EQUAL(i, msg->getRetry()); - EXPECT_EQUAL(true, msg->getRetryEnabled()); + EXPECT_EQ(i, msg->getRetry()); + EXPECT_EQ(true, msg->getRetryEnabled()); replyFromDestination(data, std::move(msg), ErrorCode::APP_TRANSIENT_ERROR, 0); msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); @@ -173,16 +150,16 @@ Test::testRetryTag(TestData &data) printf("%s", reply->getTrace().toString().c_str()); } -void -Test::testRetryEnabledTag(TestData &data) +TEST_F(ResenderTest, test_retry_enabled_tag) { + auto& data = *_data; data._retryPolicy->setEnabled(true); Message::UP msg = createMessage("msg"); msg->setRetryEnabled(false); EXPECT_TRUE(data._srcSession->send(std::move(msg), Route::parse("dst/session")).isAccepted()); msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); - EXPECT_EQUAL(false, msg->getRetryEnabled()); + EXPECT_EQ(false, msg->getRetryEnabled()); replyFromDestination(data, std::move(msg), ErrorCode::APP_TRANSIENT_ERROR, 0); Reply::UP reply = data._srcHandler.getReply(); ASSERT_TRUE(reply); @@ -192,9 +169,9 @@ Test::testRetryEnabledTag(TestData &data) printf("%s", reply->getTrace().toString().c_str()); } -void -Test::testTransientError(TestData &data) +TEST_F(ResenderTest, test_transient_error) { + auto& data = *_data; data._retryPolicy->setEnabled(true); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); @@ -211,9 +188,9 @@ Test::testTransientError(TestData &data) printf("%s", reply->getTrace().toString().c_str()); } -void -Test::testFatalError(TestData &data) +TEST_F(ResenderTest, test_fatal_error) { + auto& data = *_data; data._retryPolicy->setEnabled(true); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); @@ -227,9 +204,9 @@ Test::testFatalError(TestData &data) printf("%s", reply->getTrace().toString().c_str()); } -void -Test::testDisableRetry(TestData &data) +TEST_F(ResenderTest, test_disable_retry) { + auto& data = *_data; data._retryPolicy->setEnabled(false); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); @@ -244,16 +221,16 @@ Test::testDisableRetry(TestData &data) printf("%s", reply->getTrace().toString().c_str()); } -void -Test::testRetryDelay(TestData &data) +TEST_F(ResenderTest, test_retry_delay) { + auto& data = *_data; data._retryPolicy->setEnabled(true); data._retryPolicy->setBaseDelay(0.01); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); for (uint32_t i = 0; i < 5; ++i) { - EXPECT_EQUAL(i, msg->getRetry()); + EXPECT_EQ(i, msg->getRetry()); replyFromDestination(data, std::move(msg), ErrorCode::APP_TRANSIENT_ERROR, -1); msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); @@ -273,16 +250,16 @@ Test::testRetryDelay(TestData &data) EXPECT_TRUE(trace.find("retry 5 in 0.160") != string::npos); } -void -Test::testRequestRetryDelay(TestData &data) +TEST_F(ResenderTest, test_request_retry_delay) { + auto& data = *_data; data._retryPolicy->setEnabled(true); data._retryPolicy->setBaseDelay(1); EXPECT_TRUE(data._srcSession->send(createMessage("msg"), Route::parse("dst/session")).isAccepted()); Message::UP msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); for (uint32_t i = 0; i < 5; ++i) { - EXPECT_EQUAL(i, msg->getRetry()); + EXPECT_EQ(i, msg->getRetry()); replyFromDestination(data, std::move(msg), ErrorCode::APP_TRANSIENT_ERROR, i / 50.0); msg = data._dstHandler.getMessage(GET_MESSAGE_TIMEOUT); ASSERT_TRUE(msg); @@ -302,3 +279,4 @@ Test::testRequestRetryDelay(TestData &data) EXPECT_TRUE(trace.find("retry 5 in 0.080") != string::npos); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/messagebus/src/tests/result/CMakeLists.txt b/messagebus/src/tests/result/CMakeLists.txt index 6aba63fdc41..d4360f3826b 100644 --- a/messagebus/src/tests/result/CMakeLists.txt +++ b/messagebus/src/tests/result/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(messagebus_result_test_app TEST DEPENDS messagebus_messagebus-test messagebus + GTest::gtest ) vespa_add_test(NAME messagebus_result_test_app COMMAND messagebus_result_test_app) diff --git a/messagebus/src/tests/result/result.cpp b/messagebus/src/tests/result/result.cpp index 4140e26db95..ce6e19b7dcb 100644 --- a/messagebus/src/tests/result/result.cpp +++ b/messagebus/src/tests/result/result.cpp @@ -1,13 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/messagebus/result.h> #include <vespa/messagebus/error.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/testlib/simplemessage.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace mbus; +namespace { + struct MyMessage : public SimpleMessage { static int ctorCnt; @@ -22,28 +24,23 @@ struct MyMessage : public SimpleMessage int MyMessage::ctorCnt = 0; int MyMessage::dtorCnt = 0; -struct Test : public vespalib::TestApp -{ - Result sendOk(Message::UP msg); - Result sendFail(Message::UP msg); - int Main() override; -}; - Result -Test::sendOk(Message::UP msg) { +sendOk(Message::UP msg) +{ (void) msg; return Result(); } Result -Test::sendFail(Message::UP msg) { +sendFail(Message::UP msg) +{ return Result(Error(ErrorCode::FATAL_ERROR, "error"), std::move(msg)); } -int -Test::Main() +} + +TEST(ResultTest, test_result) { - TEST_INIT("result_test"); { // test accepted Message::UP msg(new MyMessage("test")); Result res = sendOk(std::move(msg)); @@ -68,7 +65,6 @@ Test::Main() } EXPECT_TRUE(MyMessage::ctorCnt == 2); EXPECT_TRUE(MyMessage::dtorCnt == 2); - TEST_DONE(); } -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index b4ca98f316f..b41bb29376b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -185,21 +185,34 @@ public class SimpleFeeder implements ReplyHandler { } } public void send(FeedOperation op) { - if (op.getType() == FeedOperation.Type.DOCUMENT) { - if (!isFirst) { - try { - outputStream.write(','); - outputStream.write('\n'); - } catch (IOException e) { - failure.set(e); - } - } else { - isFirst = false; + switch (op.getType()) { + case DOCUMENT -> { + addCommaAndNewline(); + writer.write(op.getDocumentPut().getDocument()); } - writer.write(op.getDocumentPut().getDocument()); + case REMOVE -> { + addCommaAndNewline(); + writer.write(op.getDocumentRemove()); + } + default -> { /* TODO: No more operations supported yet */ } } numReplies.incrementAndGet(); } + + private void addCommaAndNewline() { + if (! isFirst) { + try { + outputStream.write(','); + outputStream.write('\n'); + } catch (IOException e) { + failure.set(e); + } + } + else { + isFirst = false; + } + } + public void close() throws Exception { outputStream.write('\n'); outputStream.write(']'); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java b/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java index 582148e8eaa..28859d0e7c4 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java @@ -87,8 +87,13 @@ public class SimpleFeederTest { "", "(.+\n)+" + "\\s*\\d+,\\s*3,.+\n"); - assertEquals(58, dump.size()); - assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}}\n]", dump.toString()); + assertEquals(93, dump.size()); + assertEquals(""" + [ + {"id":"id:simple:simple::0","fields":{"my_str":"foo"}}, + {"remove":"id:simple:simple::2"} + ]""", + dump.toString()); } @Test @@ -116,8 +121,14 @@ public class SimpleFeederTest { "", "(.+\n)+" + "\\s*\\d+,\\s*3,.+\n"); - assertEquals(115, dump.size()); - assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}},\n {\"id\":\"id:simple:simple::1\",\"fields\":{\"my_str\":\"bar\"}}\n]", dump.toString()); + assertEquals(150, dump.size()); + assertEquals(""" + [ + {"id":"id:simple:simple::0","fields":{"my_str":"foo"}}, + {"id":"id:simple:simple::1","fields":{"my_str":"bar"}}, + {"remove":"id:simple:simple::2"} + ]""", + dump.toString()); assertFeed(dump.toString(), new MessageHandler() { @Override @@ -129,7 +140,7 @@ public class SimpleFeederTest { }, "", "(.+\n)+" + - "\\s*\\d+,\\s*2,.+\n"); + "\\s*\\d+,\\s*3,.+\n"); } @Test diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java index 1cc68777c88..bb540f82ea4 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java +++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java @@ -67,9 +67,12 @@ public final class TinyIdentitySet<E> implements Set<E> { * @param initSize * initial size of internal element array */ - public TinyIdentitySet(final int initSize) { + public TinyIdentitySet(int initSize) { entries = new Object[initSize]; } + public TinyIdentitySet() { + this(4); + } /** * Expose the index in the internal array of a given object. -1 is returned |