aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2024-03-17 12:35:09 +0100
committerGitHub <noreply@github.com>2024-03-17 12:35:09 +0100
commit8e2d974cc20b9b5922f93fc06d93c930bcf485fb (patch)
tree9701d8227774f4f957729b5185c9bdd51c99ac3b
parent2e3832e4cf34f65ed24485f08b24a3370b97ee07 (diff)
parent6788825d4f9f2a092af45bdf14447fa9a762151a (diff)
Merge pull request #30666 from vespa-engine/revert-30658-revert-30652-revert-30644-revert-30643-revert-30642-revert-30640-revert-30620-revert-30616-revert-30615-balder/single-searchcluster
Revert "Single searchcluster take 5"
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/Content.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java150
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java13
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java116
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java100
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java41
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java8
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java5
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java5
17 files changed, 341 insertions, 165 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
index 39894d95c2c..0d42219dade 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
@@ -30,11 +30,13 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> {
ContentCluster nextCluster = context.model().getContentClusters().get(clusterName);
if (nextCluster != null) {
- if (currentCluster.getSearch().getIndexed() != null && nextCluster.getSearch().getIndexed() != null) {
- validateStreamingCluster(currentCluster, currentCluster.getSearch().getIndexed(),
- nextCluster, nextCluster.getSearch().getIndexed())
- .forEach(context::require);
- }
+ var nextStreamingClusters = nextCluster.getSearch().getClusters();
+ currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> {
+ SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName());
+ if (nextStreamingCluster != null) {
+ validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require);
+ }
+ });
}
});
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index a71d1025ece..91622e9fe31 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -5,6 +5,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.prelude.semantics.SemanticRulesConfig;
+import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.search.dispatch.Dispatcher;
@@ -34,15 +35,15 @@ import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC
* @author gjoranv
* @author Tony Vaagenes
*/
-public class ContainerSearch extends ContainerSubsystem<SearchChains> implements
+public class ContainerSearch extends ContainerSubsystem<SearchChains>
+ implements
IndexInfoConfig.Producer,
IlscriptsConfig.Producer,
QrSearchersConfig.Producer,
QueryProfilesConfig.Producer,
SemanticRulesConfig.Producer,
PageTemplatesConfig.Producer,
- SchemaInfoConfig.Producer
-{
+ SchemaInfoConfig.Producer {
public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName();
@@ -168,7 +169,25 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> implements
@Override
public void getConfig(QrSearchersConfig.Builder builder) {
- searchClusters.forEach(sc -> builder.searchcluster(sc.getQrSearcherConfig()));
+ for (int i = 0; i < searchClusters.size(); i++) {
+ SearchCluster sys = findClusterWithId(searchClusters, i);
+ var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName());
+ for (SchemaInfo spec : sys.schemas().values()) {
+ scB.searchdef(spec.fullSchema().getName());
+ }
+ scB.rankprofiles_configid(sys.getConfigId());
+ scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName()));
+ scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec()));
+ builder.searchcluster(scB);
+ }
+ }
+
+ private static SearchCluster findClusterWithId(List<SearchCluster> clusters, int index) {
+ for (SearchCluster sys : clusters) {
+ if (sys.getClusterIndex() == index)
+ return sys;
+ }
+ throw new IllegalArgumentException("No search cluster with index " + index + " exists");
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
index 4ac7a8e442a..1254f8e110a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -135,11 +135,8 @@ public class Content extends ConfigModel {
public static List<SearchCluster> getSearchClusters(ConfigModelRepo pc) {
List<SearchCluster> clusters = new ArrayList<>();
- for (ContentCluster c : getContentClusters(pc)) {
- if (c.getSearch().hasIndexedCluster()) {
- clusters.add(c.getSearch().getIndexed());
- }
- }
+ for (ContentCluster c : getContentClusters(pc))
+ clusters.addAll(c.getSearch().getClusters().values());
return clusters;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 01708333ed5..4d6f454f7e0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.model.search.NodeSpec;
import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler;
import com.yahoo.vespa.model.search.SearchCluster;
import com.yahoo.vespa.model.search.SearchNode;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import com.yahoo.vespa.model.search.TransactionLogServer;
import com.yahoo.vespa.model.search.Tuning;
import org.w3c.dom.Element;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.LinkedHashMap;
+import java.util.Optional;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Predicate;
@@ -87,8 +89,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
public Builder(Map<String, NewDocumentType> documentDefinitions,
Set<NewDocumentType> globallyDistributedDocuments,
- double fractionOfMemoryReserved, ResourceLimits resourceLimits)
- {
+ double fractionOfMemoryReserved, ResourceLimits resourceLimits) {
this.documentDefinitions = documentDefinitions;
this.globallyDistributedDocuments = globallyDistributedDocuments;
this.fractionOfMemoryReserved = fractionOfMemoryReserved;
@@ -102,9 +103,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
Boolean flushOnShutdownElem = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown");
Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog");
- var search = new ContentSearchCluster(ancestor, clusterName, deployState.getProperties().featureFlags(),
- documentDefinitions, globallyDistributedDocuments,
- getFlushOnShutdown(flushOnShutdownElem), syncTransactionLog,
+ var search = new ContentSearchCluster(ancestor, clusterName,
+ deployState.getProperties().featureFlags(),
+ documentDefinitions,
+ globallyDistributedDocuments,
+ getFlushOnShutdown(flushOnShutdownElem),
+ syncTransactionLog,
fractionOfMemoryReserved);
ModelElement tuning = clusterElem.childByPath("engine.proton.tuning");
@@ -113,7 +117,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
search.setResourceLimits(resourceLimits);
- buildSearchCluster(deployState, clusterElem, clusterName, search);
+ buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search);
+ buildIndexedSearchCluster(deployState, clusterElem, clusterName, search);
return search;
}
@@ -125,18 +130,73 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
return clusterElem.childAsDouble("engine.proton.query-timeout");
}
- private void buildSearchCluster(DeployState deployState, ModelElement clusterElem,
- String clusterName, ContentSearchCluster search) {
+ private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) {
+ var schemas = deployState.getSchemas();
+ for (var candidate : schemas) {
+ if (candidate.getName().equals(docTypeName)) {
+ return candidate;
+ }
+ }
+ return null;
+ }
+
+ private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) {
ModelElement docElem = clusterElem.child("documents");
- if (docElem == null) return;
- Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
- if (visibilityDelay != null) {
- search.setVisibilityDelay(visibilityDelay);
+ if (docElem == null) {
+ return;
+ }
+
+ for (ModelElement docType : docElem.subElements("document")) {
+ String docTypeName = docType.stringAttribute("type");
+ String mode = docType.stringAttribute("mode");
+ var schema = findResponsibleSchema(deployState, docTypeName);
+ if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) {
+ buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType);
+ }
}
+ }
- IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
- search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document"));
+ private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName,
+ ContentSearchCluster search, ModelElement docType) {
+ String docTypeName = docType.stringAttribute("type");
+ StreamingSearchCluster cluster = new StreamingSearchCluster(search,
+ clusterName + "." + docTypeName,
+ 0,
+ docTypeName,
+ clusterName);
+ search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType));
+ }
+
+ private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem,
+ String clusterName, ContentSearchCluster search) {
+ List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem);
+ if (!indexedDefs.isEmpty()) {
+ IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
+
+ Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
+ if (visibilityDelay != null) {
+ search.setVisibilityDelay(visibilityDelay);
+ }
+
+ search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs);
+ }
+ }
+
+ private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) {
+ List<ModelElement> indexedDefs = new ArrayList<>();
+ ModelElement docElem = clusterElem.child("documents");
+ if (docElem == null) {
+ return indexedDefs;
+ }
+
+ for (ModelElement docType : docElem.subElements("document")) {
+ String mode = docType.stringAttribute("mode");
+ if ("index".equals(mode)) {
+ indexedDefs.add(docType);
+ }
+ }
+ return indexedDefs;
}
}
@@ -187,12 +247,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
cluster.setQueryTimeout(queryTimeout);
}
cluster.deriveFromSchemas(deployState);
- if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) {
- addCluster(cluster);
- }
+ addCluster(cluster);
}
-
private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) {
for (ModelElement e : searchDefs) {
SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e);
@@ -200,7 +257,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
if (schema == null)
throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " +
this + " does not exist");
- if (schema.isDocumentsOnly()) continue;
sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null));
}
@@ -226,20 +282,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
* with indexing, null if it has both or none.
*/
public Boolean isStreaming() {
- if (indexedCluster == null) return false;
- boolean hasStreaming = indexedCluster.hasStreaming();
- if (indexedCluster.hasIndexed() == hasStreaming) return null;
+ boolean hasStreaming = false;
+ boolean hasIndexed = false;
+ for (var cluster : clusters.values()) {
+ if (cluster.hasStreaming())
+ hasStreaming = true;
+ else
+ hasIndexed = true;
+ }
+ if (hasIndexed == hasStreaming) return null;
return hasStreaming;
}
- public boolean hasStreaming() {
- return (indexedCluster != null) && indexedCluster.hasStreaming();
- }
-
- public boolean hasIndexed() {
- return (indexedCluster != null) && indexedCluster.hasIndexed();
- }
-
public List<SearchNode> getSearchNodes() {
return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed;
}
@@ -302,6 +356,21 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
this.redundancy = redundancy;
}
+ private Optional<StreamingSearchCluster> findStreamingCluster(String docType) {
+ return getClusters().values().stream()
+ .filter(StreamingSearchCluster.class::isInstance)
+ .map(StreamingSearchCluster.class::cast)
+ .filter(ssc -> ssc.schemas().get(docType) != null)
+ .findFirst();
+ }
+
+ public List<StreamingSearchCluster> getStreamingClusters() {
+ return getClusters().values().stream()
+ .filter(StreamingSearchCluster.class::isInstance)
+ .map(StreamingSearchCluster.class::cast)
+ .toList();
+ }
+
public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); }
public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); }
public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); }
@@ -313,13 +382,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
private boolean hasIndexingModeStreaming(NewDocumentType type) {
- if (indexedCluster == null) return false;
- return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING;
+ return findStreamingCluster(type.getFullName().getName()).isPresent();
}
private boolean hasIndexingModeIndexed(NewDocumentType type) {
- if (indexedCluster == null) return false;
- return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX;
+ return !hasIndexingModeStreaming(type)
+ && hasIndexedCluster()
+ && getIndexed().hasDocumentDB(type.getFullName().getName());
}
private boolean hasIndexingModeStoreOnly(NewDocumentType type) {
@@ -328,7 +397,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
@Override
public void getConfig(ProtonConfig.Builder builder) {
- boolean hasAnyNonIndexedSchema = false;
+ boolean hasAnyNonIndexedCluster = false;
for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) {
ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
String docTypeName = type.getFullName().getName();
@@ -340,13 +409,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers);
if (hasIndexingModeStreaming(type)) {
- hasAnyNonIndexedSchema = true;
- indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ hasAnyNonIndexedCluster = true;
+ findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING);
} else if (hasIndexingModeIndexed(type)) {
- indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
} else {
- hasAnyNonIndexedSchema = true;
+ hasAnyNonIndexedCluster = true;
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY);
}
if (globalDocType) {
@@ -355,7 +424,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
builder.documentdb(ddbB);
}
- if (hasAnyNonIndexedSchema) {
+ if (hasAnyNonIndexedCluster) {
builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2));
} else {
builder.feeding.concurrency(defaultFeedConcurrency);
@@ -409,6 +478,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
getIndexed().getConfig(builder);
}
}
+ public Map<String, SearchCluster> getClusters() { return clusters; }
public IndexedSearchCluster getIndexed() { return indexedCluster; }
public boolean hasIndexedCluster() { return indexedCluster != null; }
public IndexingDocproc getIndexingDocproc() { return indexingDocproc; }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 5a9b4bb0760..791faa401ed 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -140,7 +140,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
c.search.handleRedundancy(c.redundancy);
setupSearchCluster(c.search, contentElement, deployState.getDeployLogger());
- if (c.search.hasIndexed() && !(c.persistenceFactory instanceof ProtonEngine.Factory) )
+ if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) )
throw new IllegalArgumentException("Indexed search requires proton as engine");
if (documentsElement != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java
index f4e6244fa25..7fc713f81ef 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java
@@ -13,7 +13,7 @@ public class EngineFactoryBuilder {
public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) {
ModelElement persistence = clusterElem.child("engine");
if (persistence != null) {
- if (c.getSearch().hasIndexed() && persistence.child("proton") == null) {
+ if (c.getSearch().hasIndexedCluster() && persistence.child("proton") == null) {
throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine.");
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 67d99e300de..4a37b27d1c7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -45,6 +45,9 @@ public class IndexedSearchCluster extends SearchCluster implements
summaryDecodePolicy = featureFlags.summaryDecodePolicy();
}
+ @Override
+ protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; }
+
public void addSearcher(SearchNode searcher) {
searchNodes.add(searcher);
}
@@ -57,6 +60,16 @@ public class IndexedSearchCluster extends SearchCluster implements
}
public Tuning getTuning() { return tuning; }
+ @Override
+ public void deriveFromSchemas(DeployState deployState) {
+ for (SchemaInfo spec : schemas().values()) {
+ if (spec.fullSchema() instanceof DocumentOnlySchema) continue;
+ var db = new DocumentDatabase(this, spec.fullSchema().getName(),
+ new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()));
+ add(db);
+ }
+ }
+
public void setSearchCoverage(SearchCoverage searchCoverage) {
this.searchCoverage = searchCoverage;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index 9d459259253..fc4b96ec384 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -3,32 +3,18 @@ package com.yahoo.vespa.model.search;
import com.yahoo.config.ConfigInstance;
import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.container.QrSearchersConfig;
-import com.yahoo.schema.DocumentOnlySchema;
-import com.yahoo.schema.derived.AttributeFields;
-import com.yahoo.schema.derived.DerivedConfiguration;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
-import com.yahoo.vespa.config.search.RankProfilesConfig;
-import com.yahoo.vespa.config.search.SummaryConfig;
-import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
-import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
-import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
-import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
-import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
-import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.config.model.producer.AnyConfigProducer;
import com.yahoo.config.model.producer.TreeConfigProducer;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -50,50 +36,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
private Double visibilityDelay = 0.0;
private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>();
private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>();
- private final Map<String, AttributesProducer> documentDBProducerForStreaming = new HashMap<>();
- private final List<LegacyStreamingProxy> legacyproxy = new ArrayList<>();
-
- private static class LegacyStreamingProxy extends TreeConfigProducer<AnyConfigProducer> implements
- AttributesConfig.Producer,
- RankProfilesConfig.Producer,
- RankingConstantsConfig.Producer,
- RankingExpressionsConfig.Producer,
- OnnxModelsConfig.Producer,
- JuniperrcConfig.Producer,
- SummaryConfig.Producer,
- VsmsummaryConfig.Producer,
- VsmfieldsConfig.Producer
- {
- private final DocumentDatabase db;
- LegacyStreamingProxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName,
- String schemaName, DerivedConfiguration derived) {
- super(parent, "cluster." + clusterName + "." + schemaName);
- this.db = new DocumentDatabase(this, schemaName, derived);
- }
- @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); }
- }
-
- private static class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer {
- private final DerivedConfiguration derived;
-
- AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType, DerivedConfiguration derived) {
- super(parent, docType);
- this.derived = derived;
- }
-
- @Override
- public void getConfig(AttributesConfig.Builder builder) {
- derived.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS);
- }
- }
public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) {
super(parent, "cluster." + clusterName);
@@ -106,6 +48,9 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
public void add(SchemaInfo schema) {
schemas.put(schema.name(), schema);
}
+ public void add(DocumentDatabase db) {
+ documentDbs.put(db.getName(), db);
+ }
public boolean hasDocumentDB(String name) {
return documentDbs.containsKey(name);
@@ -127,19 +72,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
* Derives the schemas from the application package.
* Also stores the document names contained in the schemas.
*/
- public void deriveFromSchemas(DeployState deployState) {
- for (SchemaInfo spec : schemas().values()) {
- if (spec.fullSchema() instanceof DocumentOnlySchema) continue; // TODO verify if this special handling is necessary
- String schemaName = spec.fullSchema().getName();
- var derived = new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode());
- documentDbs.put(schemaName, new DocumentDatabase(this, schemaName, derived));
- if (spec.getIndexMode() == SchemaInfo.IndexMode.STREAMING) {
- var parent = (TreeConfigProducer<AnyConfigProducer>)getParent();
- documentDBProducerForStreaming.put(schemaName, new AttributesProducer(parent, schemaName, derived));
- legacyproxy.add(new LegacyStreamingProxy(parent, clusterName, schemaName, derived));
- }
- }
- }
+ public abstract void deriveFromSchemas(DeployState deployState);
/** Returns the document databases contained in this cluster */
public List<DocumentDatabase> getDocumentDbs() {
@@ -147,6 +80,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
}
public String getClusterName() { return clusterName; }
+ public final String getIndexingModeName() { return getIndexingMode().getName(); }
public final boolean hasStreaming() {
return schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING);
}
@@ -157,6 +91,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
public final void setQueryTimeout(Double to) { this.queryTimeout = to; }
public final void setVisibilityDelay(double delay) { this.visibilityDelay = delay; }
+ protected abstract IndexingMode getIndexingMode();
public final Double getVisibilityDelay() { return visibilityDelay; }
public final Double getQueryTimeout() { return queryTimeout; }
public final void setClusterIndex(int index) { this.index = index; }
@@ -165,26 +100,13 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) {
DocumentDatabase db = documentDbs.get(documentType);
if (db != null) {
- builder.inputdoctypename(documentType);
- if (db.getDerivedConfiguration().isStreaming()) {
- builder.configid(documentDBProducerForStreaming.get(documentType).getConfigId());
- } else {
- builder.configid(db.getConfigId());
- }
+ fillDocumentDBConfig(db, builder);
}
}
- public QrSearchersConfig.Searchcluster.Builder getQrSearcherConfig() {
- var builder = new QrSearchersConfig.Searchcluster.Builder()
- .name(getClusterName())
- .rankprofiles_configid(getConfigId())
- .storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(getStorageRouteSpec()))
- .indexingmode(hasStreaming() ? QrSearchersConfig.Searchcluster.Indexingmode.STREAMING
- : QrSearchersConfig.Searchcluster.Indexingmode.REALTIME);
- for (SchemaInfo spec : schemas().values()) {
- builder.searchdef(spec.fullSchema().getName());
- }
- return builder;
+ protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
+ ddbB.inputdoctypename(sdoc.getSchemaName())
+ .configid(sdoc.getConfigId());
}
@Override
@@ -220,6 +142,24 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
@Override
public String toString() { return "search-capable cluster '" + clusterName + "'"; }
+ public static final class IndexingMode {
+
+ public static final IndexingMode REALTIME = new IndexingMode("REALTIME");
+ public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
+
+ private final String name;
+
+ private IndexingMode(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+
+ public String toString() {
+ return "indexingmode: " + name;
+ }
+ }
+
/**
* Class used to retrieve combined configuration from multiple document databases.
* It is not a direct {@link ConfigInstance.Producer} of those configs,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
new file mode 100644
index 00000000000..e188a086614
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -0,0 +1,100 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AnyConfigProducer;
+import com.yahoo.config.model.producer.TreeConfigProducer;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.derived.AttributeFields;
+import com.yahoo.schema.derived.DerivedConfiguration;
+import com.yahoo.schema.derived.SchemaInfo;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
+
+/**
+ * A search cluster of type streaming.
+ *
+ * @author baldersheim
+ * @author vegardh
+ */
+public class StreamingSearchCluster extends SearchCluster implements
+ AttributesConfig.Producer,
+ RankProfilesConfig.Producer,
+ RankingConstantsConfig.Producer,
+ RankingExpressionsConfig.Producer,
+ OnnxModelsConfig.Producer,
+ JuniperrcConfig.Producer,
+ SummaryConfig.Producer,
+ VsmsummaryConfig.Producer,
+ VsmfieldsConfig.Producer
+{
+ private final String storageRouteSpec;
+ private final AttributesProducer attributesConfig;
+ private final String docTypeName;
+
+ public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index,
+ String docTypeName, String storageRouteSpec) {
+ super(parent, clusterName, index);
+ attributesConfig = new AttributesProducer(parent, docTypeName);
+ this.docTypeName = docTypeName;
+ this.storageRouteSpec = storageRouteSpec;
+ }
+
+ @Override
+ protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; }
+ public final String getStorageRouteSpec() { return storageRouteSpec; }
+
+ public String getDocTypeName() { return docTypeName; }
+
+ public DerivedConfiguration derived() { return db().getDerivedConfiguration(); }
+
+ @Override
+ public void deriveFromSchemas(DeployState deployState) {
+ if (schemas().isEmpty()) return;
+ if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size());
+
+ Schema schema = schemas().values().stream().findAny().get().fullSchema();
+ if ( ! schema.getName().equals(docTypeName))
+ throw new IllegalArgumentException("Document type name '" + docTypeName +
+ "' must be the same as the schema name '" + schema.getName() + "'");
+ add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING)));
+ }
+
+ protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
+ super.fillDocumentDBConfig(sdoc, ddbB);
+ ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up
+ }
+
+ private DocumentDatabase db() { return getDocumentDbs().get(0); }
+
+ // These are temporary until backend uses correct config id.
+ @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);}
+
+ private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer {
+
+ AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) {
+ super(parent, docType);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS);
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
index 2879baf7b88..d3e62dae947 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
@@ -29,7 +29,7 @@ public class StreamingValidatorTest {
new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/")
.create();
});
- assertTrue(exception.getMessage().contains("For search cluster 'content', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
+ assertTrue(exception.getMessage().contains("For search cluster 'content.ad', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
"Document references and imported fields are not allowed in streaming search."));
}
@@ -52,7 +52,7 @@ public class StreamingValidatorTest {
"attribute { distance-metric: euclidean } }");
var warnings = filter(logger.warnings);
assertEquals(1, warnings.size());
- assertEquals("For search cluster 'content', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
+ assertEquals("For search cluster 'content.test', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
warnings.get(0));
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
index 28eabd18539..ee64ceb6969 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
@@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest {
private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }";
private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }";
private static final List<ServiceInfo> FOO_SERVICE =
- List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null"));
+ List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null"));
private static final List<ServiceInfo> BAR_SERVICE =
- List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null"));
+ List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null"));
@Test
void changing_field_type_requires_refeed() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index a4a5f4cb4df..d3e69e64fe6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -63,6 +63,7 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s = a.getSearch();
assertFalse(s.hasIndexedCluster());
+ assertTrue(s.getClusters().isEmpty());
assertTrue(a.getPersistence() instanceof com.yahoo.vespa.model.content.engines.DummyPersistence.Factory);
}
@@ -82,6 +83,7 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s = a.getSearch();
assertFalse(s.hasIndexedCluster());
+ assertTrue(s.getClusters().isEmpty());
assertTrue(a.getPersistence() instanceof ProtonEngine.Factory);
@@ -106,6 +108,7 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s = a.getSearch();
assertFalse(s.hasIndexedCluster());
+ assertTrue(s.getClusters().isEmpty());
assertTrue(a.getPersistence() instanceof ProtonEngine.Factory);
@@ -127,6 +130,7 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s = a.getSearch();
assertFalse(s.hasIndexedCluster());
+ assertTrue(s.getClusters().isEmpty());
assertNull(s.getIndexed());
assertNull(a.getRootGroup().getName());
@@ -147,6 +151,7 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentCluster c = CollectionUtil.first(m.getContentClusters().values());
ContentSearchCluster s = c.getSearch();
assertTrue(s.hasIndexedCluster());
+ assertEquals(1, s.getClusters().size());
assertNotNull(s.getIndexed());
assertEquals("clu", s.getIndexed().getClusterName());
assertEquals(7.3, s.getIndexed().getQueryTimeout(), 0.0);
@@ -191,9 +196,11 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s;
s = cluster.getSearch();
- assertTrue(s.hasIndexedCluster());
- SearchCluster sc = s.getIndexed();
- assertEquals(musicClusterId, sc.getClusterName());
+ assertFalse(s.hasIndexedCluster());
+ assertEquals(1, s.getClusters().size());
+ assertNull(s.getIndexed());
+ SearchCluster sc = s.getClusters().get(musicClusterId + ".music");
+ assertEquals(musicClusterId + ".music", sc.getClusterName());
assertEquals(musicClusterId, sc.getStorageRouteSpec());
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
@@ -229,11 +236,24 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\"/>" +
" </group>" +
"</content>");
- ContentSearchCluster s = cluster.getSearch();
- assertTrue(s.hasIndexedCluster());
- assertEquals(2, s.getIndexed().getDocumentDbs().size());
- assertTrue(s.getIndexed().hasDocumentDB("book"));
- assertTrue(s.getIndexed().hasDocumentDB("music"));
+ ContentSearchCluster s;
+
+ s = cluster.getSearch();
+ assertFalse(s.hasIndexedCluster());
+ assertEquals(2, s.getClusters().size());
+ assertNull(s.getIndexed());
+ {
+ String id = musicClusterId + ".book";
+ SearchCluster sc = s.getClusters().get(id);
+ assertEquals(id, sc.getClusterName());
+ assertEquals(musicClusterId, sc.getStorageRouteSpec());
+ }
+ {
+ String id = musicClusterId + ".music";
+ SearchCluster sc = s.getClusters().get(id);
+ assertEquals(id, sc.getClusterName());
+ assertEquals(musicClusterId, sc.getStorageRouteSpec());
+ }
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
assertEquals(1, cluster.getStorageCluster().getChildren().size());
@@ -257,6 +277,7 @@ public class ContentBuilderTest extends DomBuilderTest {
s = b.getSearch();
assertTrue(s.hasIndexedCluster());
+ assertEquals(1, s.getClusters().size());
assertNotNull(s.getIndexed());
assertEquals("b", s.getIndexed().getClusterName());
@@ -341,7 +362,9 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
" </group>" +
"</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2")));
- ContentSearchCluster s = b.getSearch();
+ ContentSearchCluster s;
+
+ s = b.getSearch();
assertTrue(s.hasIndexedCluster());
assertNotNull(s.getIndexed());
assertEquals(1, b.getStorageCluster().getChildren().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java
index 02a7a946e17..8f02cf7a6d4 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.search.searchchain;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
@@ -27,13 +28,18 @@ public class MockSearchClusters {
@Override
public void deriveFromSchemas(DeployState deployState) { }
- @Override public List<DocumentDatabase> getDocumentDbs() { return List.of(); }
+ @Override public List<DocumentDatabase> getDocumentDbs() {return List.of();}
@Override public void getConfig(AttributesConfig.Builder builder) {}
@Override public void getConfig(DocumentdbInfoConfig.Builder builder) {}
@Override public void getConfig(IndexInfoConfig.Builder builder) {}
@Override public void getConfig(IlscriptsConfig.Builder builder) {}
@Override public void getConfig(SchemaInfoConfig.Builder builder) {}
+ @Override
+ protected SearchCluster.IndexingMode getIndexingMode() {
+ return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME;
+ }
+
}
public static SearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index ae08f78c404..6f9127779db 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -163,6 +163,7 @@ public class IndexedTest extends ContentBaseTest {
// "transactionlogserver"};
// DomContentBuilderTest.assertServices(h, expectedServices);
ContentCluster s = model.getContentClusters().get("test");
+ assertFalse(s.getSearch().hasIndexedCluster());
StorServerConfig.Builder builder = new StorServerConfig.Builder();
s.getStorageCluster().getConfig(builder);
@@ -174,6 +175,7 @@ public class IndexedTest extends ContentBaseTest {
VespaModel model = getStreamingVespaModel();
ContentCluster s = model.getContentClusters().get("test");
assertNotNull(s);
+ assertFalse(s.getSearch().hasIndexedCluster());
ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID);
assertEquals(1, config.storage().size());
assertEquals("test", config.storage(0).name());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index b4c625a599b..d3741405c15 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -309,7 +309,7 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModelWithMode(mode, sds);
- DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
+ DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
assertEquals(1, dcfg.documentdb().size());
DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0);
assertEquals("type", db.name());
@@ -328,11 +328,13 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModel(sds, "");
DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
- assertEquals(2, indexed_cfg.documentdb().size());
+ assertEquals(1, indexed_cfg.documentdb().size());
var db = indexed_cfg.documentdb(0);
assertEquals("a", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode());
- db = indexed_cfg.documentdb(1);
+ DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b");
+ assertEquals(1, streaming_cfg.documentdb().size());
+ db = streaming_cfg.documentdb(0);
assertEquals("b", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode());
}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
index d4e295fb51f..d61da0ebedf 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
@@ -55,7 +55,7 @@ public class GroupingValidator extends Searcher {
var searchCluster = qrsConfig.searchcluster(clusterId);
QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode = searchCluster.indexingmode();
enabled = (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING);
- clusterName = searchCluster.name();
+ clusterName = enabled ? searchCluster.name() : null;
for (AttributesConfig.Attribute attr : attributesConfig.attribute()) {
attributes.put(attr.name(), attr);
}
@@ -97,7 +97,8 @@ public class GroupingValidator extends Searcher {
@Override
public void visitExpression(GroupingExpression exp) {
- if (exp instanceof AttributeMapLookupValue mapLookup) {
+ if (exp instanceof AttributeMapLookupValue) {
+ AttributeMapLookupValue mapLookup = (AttributeMapLookupValue) exp;
verifyHasAttribute(mapLookup.getKeyAttribute());
verifyHasAttribute(mapLookup.getValueAttribute());
if (mapLookup.hasKeySourceAttribute()) {
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
index 1c95a07cdfd..033e567d53a 100644
--- a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
@@ -11,6 +11,7 @@ import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.Searcher;
import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.FieldInfo;
import com.yahoo.search.schema.SchemaInfo;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.search.searchchain.PhaseNames;
@@ -54,7 +55,7 @@ public class QueryValidator extends Searcher {
public boolean visit(Item item) {
if (item instanceof HasIndexItem indexItem) {
var field = schema.fieldInfo(indexItem.getIndexName());
- if (field.isEmpty()) return true;
+ if (! field.isPresent()) return true;
if (field.get().type().kind() == Field.Type.Kind.TENSOR)
throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() +
"': It is a tensor field");
@@ -75,7 +76,7 @@ public class QueryValidator extends Searcher {
if (schema.isStreaming()) return true; // prefix is always supported
if (item instanceof PrefixItem prefixItem) {
var field = schema.fieldInfo(prefixItem.getIndexName());
- if (field.isEmpty()) return true;
+ if (! field.isPresent()) return true;
if ( ! field.get().isAttribute())
throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is not an attribute field: Prefix matching is not supported");
if (field.get().isIndex()) // index overrides attribute