aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2024-03-13 22:31:36 +0100
committerGitHub <noreply@github.com>2024-03-13 22:31:36 +0100
commit14ebb1d9b9e35ac7b853f0fe919826899dd08d2f (patch)
treec00562090b5bb8bb55da461e93f085d6910ad5a6 /config-model
parent7f94821395235f2bb75de793fb077622327c359d (diff)
Revert "Balder/single searchcluster"
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java23
-rw-r--r--config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java142
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java89
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java99
-rw-r--r--config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java12
-rw-r--r--config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java37
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java20
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java4
22 files changed, 374 insertions, 234 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
index a7f66966538..b35918b3460 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
@@ -28,7 +28,7 @@ import java.io.Writer;
public class DerivedConfiguration {
private final Schema schema;
- private final SchemaInfo.IndexMode indexMode;
+ private final boolean isStreaming;
private Summaries summaries;
private Juniperrc juniperrc;
private AttributeFields attributeFields;
@@ -57,8 +57,7 @@ public class DerivedConfiguration {
}
DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) {
- this(new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(),
- schema, SchemaInfo.IndexMode.INDEX);
+ this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false);
}
/**
@@ -68,8 +67,8 @@ public class DerivedConfiguration {
* argument is live. Which means that this object will be inconsistent if the given
* schema is later modified.
*/
- public DerivedConfiguration(DeployState deployState, Schema schema, SchemaInfo.IndexMode indexMode) {
- this.indexMode = indexMode;
+ public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) {
+ this.isStreaming = isStreaming;
try {
Validator.ensureNotNull("Schema", schema);
this.schema = schema;
@@ -82,9 +81,9 @@ public class DerivedConfiguration {
summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags());
juniperrc = new Juniperrc(schema);
rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState);
- indexingScript = new IndexingScript(schema, isStreaming());
- indexInfo = new IndexInfo(schema, isStreaming());
- schemaInfo = new SchemaInfo(schema, indexMode, deployState.rankProfileRegistry(), summaries);
+ indexingScript = new IndexingScript(schema, isStreaming);
+ indexInfo = new IndexInfo(schema, isStreaming);
+ schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries);
indexSchema = new IndexSchema(schema);
importedFields = new ImportedFields(schema);
}
@@ -154,7 +153,7 @@ public class DerivedConfiguration {
}
public boolean isStreaming() {
- return indexMode == SchemaInfo.IndexMode.STREAMING;
+ return isStreaming;
}
public Summaries getSummaries() {
@@ -166,11 +165,7 @@ public class DerivedConfiguration {
}
public void getConfig(AttributesConfig.Builder builder) {
- if (isStreaming()) {
- getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS);
- } else {
- getConfig(builder, AttributeFields.FieldSet.ALL);
- }
+ getConfig(builder, AttributeFields.FieldSet.ALL);
}
public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
index f996b2624db..19a045ac444 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
@@ -42,32 +42,13 @@ public final class SchemaInfo extends Derived {
private final Map<String, RankProfileInfo> rankProfiles;
private final Summaries summaries;
- private final IndexMode indexMode;
- public enum IndexMode {INDEX, STREAMING, STORE_ONLY}
-
- public SchemaInfo(Schema schema, String indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
- this(schema, indexMode(indexMode), rankProfileRegistry, summaries);
- }
- public SchemaInfo(Schema schema, IndexMode indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
+ public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
this.schema = schema;
this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema)));
this.summaries = summaries;
- this.indexMode = indexMode;
}
- private static IndexMode indexMode(String mode) {
- if (mode == null) return IndexMode.INDEX;
- return switch (mode) {
- case "index" -> IndexMode.INDEX;
- case "streaming" -> IndexMode.STREAMING;
- case "store-only" -> IndexMode.STORE_ONLY;
- default -> IndexMode.STORE_ONLY;
- };
- }
-
- public IndexMode getIndexMode() { return indexMode; }
-
public String name() { return schema.getName(); }
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
index 7b7c92a43e0..12f3f025996 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
@@ -28,10 +28,10 @@ public class ComplexFieldsWithStructFieldAttributesValidator implements Validato
public void validate(Context context) {
List<SearchCluster> searchClusters = context.model().getSearchClusters();
for (SearchCluster cluster : searchClusters) {
+ if (cluster.isStreaming()) continue;
+
for (SchemaInfo spec : cluster.schemas().values()) {
- if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
- validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger());
- }
+ validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
index 2d553c84f48..3dbba081400 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.model.application.validation;
import com.yahoo.schema.Schema;
-import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.vespa.model.application.validation.Validation.Context;
@@ -25,10 +24,11 @@ public class ComplexFieldsWithStructFieldIndexesValidator implements Validator {
@Override
public void validate(Context context) {
for (var cluster : context.model().getSearchClusters()) {
+ if (cluster.isStreaming()) {
+ continue;
+ }
for (var spec : cluster.schemas().values()) {
- if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
- validateComplexFields(context, cluster.getClusterName(), spec.fullSchema());
- }
+ validateComplexFields(context, cluster.getClusterName(), spec.fullSchema());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
index e1c64422653..15d293e4abc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
@@ -23,9 +23,10 @@ public class NoPrefixForIndexes implements Validator {
@Override
public void validate(Context context) {
for (SearchCluster cluster : context.model().getSearchClusters()) {
- for (DocumentDatabase docDb : cluster.getDocumentDbs()) {
- DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
- if ( ! sdConfig.isStreaming() ) {
+ if (cluster instanceof IndexedSearchCluster) {
+ IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+ for (DocumentDatabase docDb : sc.getDocumentDbs()) {
+ DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
Schema schema = sdConfig.getSchema();
for (ImmutableSDField field : schema.allConcreteFields()) {
if (field.doesIndexing()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 736665b2184..03076b64d0c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -69,7 +69,7 @@ public class RankSetupValidator implements Validator {
String schemaDir = clusterDir + schemaName + "/";
writeConfigs(schemaDir, docDb);
writeExtraVerifyRankSetupConfig(schemaDir, docDb);
- if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, docDb.getDerivedConfiguration().isStreaming())) {
+ if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, sc.isStreaming())) {
return;
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index cd8094f8c12..6e21adc4fe4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -31,11 +31,15 @@ public class SearchDataTypeValidator implements Validator {
public void validate(Context context) {
List<SearchCluster> clusters = context.model().getSearchClusters();
for (SearchCluster cluster : clusters) {
+ if (cluster.isStreaming()) {
+ continue;
+ }
for (SchemaInfo spec : cluster.schemas().values()) {
SDDocumentType docType = spec.fullSchema().getDocument();
- if (docType != null && spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
- validateDocument(context, cluster, spec.fullSchema(), docType);
+ if (docType == null) {
+ continue;
}
+ validateDocument(context, cluster, spec.fullSchema(), docType);
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
index 0e9d3bf7016..c0ad55fc8f4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
@@ -6,14 +6,12 @@ import com.yahoo.document.DataType;
import com.yahoo.document.NumericDataType;
import com.yahoo.document.TensorDataType;
import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.schema.Schema;
-import com.yahoo.schema.derived.DerivedConfiguration;
-import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.schema.document.MatchType;
import com.yahoo.vespa.model.application.validation.Validation.Context;
import com.yahoo.vespa.model.search.SearchCluster;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import java.util.List;
import java.util.logging.Level;
@@ -27,21 +25,20 @@ public class StreamingValidator implements Validator {
public void validate(Context context) {
List<SearchCluster> searchClusters = context.model().getSearchClusters();
for (SearchCluster cluster : searchClusters) {
- for (SchemaInfo schemaInfo : cluster.schemas().values()) {
- if (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING) {
- var deployLogger = context.deployState().getDeployLogger();
- warnStreamingAttributes(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger);
- warnStreamingGramMatching(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger);
- failStreamingDocumentReferences(cluster.getClusterName(), cluster.getDocumentDB(schemaInfo.name()).getDerivedConfiguration(), context);
- }
- }
+ if ( ! cluster.isStreaming()) continue;
+
+ var streamingCluster = (StreamingSearchCluster)cluster;
+ warnStreamingAttributes(streamingCluster, context.deployState().getDeployLogger());
+ warnStreamingGramMatching(streamingCluster, context.deployState().getDeployLogger());
+ failStreamingDocumentReferences(context, streamingCluster);
}
}
- private static void warnStreamingGramMatching(String cluster, Schema schema, DeployLogger logger) {
- for (ImmutableSDField sd : schema.allConcreteFields()) {
+ private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) {
+ for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) {
if (sd.getMatching().getType() == MatchType.GRAM) {
- logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + "', schema '" + schema.getName() +
+ logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" +
+ sc.getClusterName() +
"', SD field '" + sd.getName() +
"': n-gram matching is not supported for streaming search.");
}
@@ -50,16 +47,19 @@ public class StreamingValidator implements Validator {
/**
* Warn if one or more attributes are defined in a streaming search cluster SD.
+ *
+ * @param sc a search cluster to be checked for attributes in streaming search
+ * @param logger a DeployLogger
*/
- private static void warnStreamingAttributes(String cluster, Schema schema, DeployLogger logger) {
- for (ImmutableSDField sd : schema.allConcreteFields()) {
+ private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) {
+ for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) {
if (sd.doesAttributing()) {
- warnStreamingAttribute(cluster, schema.getName(), sd, logger);
+ warnStreamingAttribute(sc, sd, logger);
}
}
}
- private static void warnStreamingAttribute(String cluster, String schema, ImmutableSDField sd, DeployLogger logger) {
+ private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) {
// If the field is numeric, we can't print this, because we may have converted the field to
// attribute indexing ourselves (IntegerIndex2Attribute)
if (sd.getDataType() instanceof NumericDataType) return;
@@ -68,25 +68,25 @@ public class StreamingValidator implements Validator {
for (var fieldAttribute : sd.getAttributes().values()) {
if (fieldAttribute.hnswIndexParams().isPresent()) {
logger.logApplicationPackage(Level.WARNING,
- "For search cluster '" + cluster + "', schema '" + schema +
+ "For streaming search cluster '" + sc.getClusterName() +
"', SD field '" + sd.getName() +
"': hnsw index is not relevant and not supported, ignoring setting");
}
}
return;
}
- logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster +
+ logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() +
"', SD field '" + sd.getName() +
"': 'attribute' has same match semantics as 'index'.");
}
- private static void failStreamingDocumentReferences(String cluster, DerivedConfiguration derived, Context context) {
- for (Attribute attribute : derived.getAttributeFields().attributes()) {
+ private static void failStreamingDocumentReferences(Context context, StreamingSearchCluster sc) {
+ for (Attribute attribute : sc.derived().getAttributeFields().attributes()) {
DataType dataType = attribute.getDataType();
if (dataType instanceof NewDocumentReferenceDataType) {
- String errorMessage = String.format("For search cluster '%s', schema '%s': Attribute '%s' has type '%s'. " +
+ String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " +
"Document references and imported fields are not allowed in streaming search.",
- cluster, derived.getSchema().getName(), attribute.getName(), dataType.getName());
+ sc.getClusterName(), attribute.getName(), dataType.getName());
context.illegal(errorMessage);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
index 0d42219dade..3b89467299d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
@@ -1,6 +1,7 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.application.validation.change;
+import com.yahoo.config.model.api.ConfigChangeAction;
import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.documentmodel.NewDocumentType;
@@ -11,11 +12,11 @@ import com.yahoo.vespa.model.application.validation.Validation.ChangeContext;
import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder;
import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
-import com.yahoo.vespa.model.search.DocumentDatabase;
-import com.yahoo.vespa.model.search.SearchCluster;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -30,47 +31,39 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> {
ContentCluster nextCluster = context.model().getContentClusters().get(clusterName);
if (nextCluster != null) {
- var nextStreamingClusters = nextCluster.getSearch().getClusters();
- currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> {
- SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName());
- if (nextStreamingCluster != null) {
- validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require);
- }
+ List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters();
+ currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> {
+ Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters);
+ nextStreamingCluster.ifPresent(streamingSearchCluster -> validateStreamingCluster(currentCluster, currentStreamingCluster,
+ nextCluster, streamingSearchCluster).forEach(context::require));
});
}
});
}
- private static List<VespaConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster,
- SearchCluster currentStreamingCluster,
- ContentCluster nextCluster,
- SearchCluster nextStreamingCluster) {
- List<VespaConfigChangeAction> result = new ArrayList<>();
-
- for (DocumentDatabase currentDB : currentStreamingCluster.getDocumentDbs()) {
- DocumentDatabase nextDB = nextStreamingCluster.getDocumentDB(currentDB.getName());
- if (nextDB != null) {
- result.addAll(validateDocumentDB(currentCluster, currentDB, nextCluster, nextDB));
- }
- }
- return result;
+ private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) {
+ return clusters.stream()
+ .filter(cluster -> cluster.getClusterName().equals(clusterName))
+ .findFirst();
}
- private static List<VespaConfigChangeAction> validateDocumentDB(ContentCluster currentCluster, DocumentDatabase currentDB,
- ContentCluster nextCluster, DocumentDatabase nextDB) {
+ private static List<ConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster,
+ StreamingSearchCluster currentStreamingCluster,
+ ContentCluster nextCluster,
+ StreamingSearchCluster nextStreamingCluster) {
List<VespaConfigChangeAction> result = new ArrayList<>();
result.addAll(validateDocumentTypeChanges(currentCluster.id(),
- getDocumentType(currentCluster, currentDB),
- getDocumentType(nextCluster, nextDB)));
+ getDocumentType(currentCluster, currentStreamingCluster),
+ getDocumentType(nextCluster, nextStreamingCluster)));
result.addAll(validateAttributeFastAccessAdded(currentCluster.id(),
- currentDB.getDerivedConfiguration().getAttributeFields(),
- nextDB.getDerivedConfiguration().getAttributeFields()));
+ currentStreamingCluster.derived().getAttributeFields(),
+ nextStreamingCluster.derived().getAttributeFields()));
result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(),
- currentDB.getDerivedConfiguration().getAttributeFields(),
- nextDB.getDerivedConfiguration().getAttributeFields()));
+ currentStreamingCluster.derived().getAttributeFields(),
+ nextStreamingCluster.derived().getAttributeFields()));
- return modifyActions(result, getSearchNodeServices(nextCluster), nextDB.getName());
+ return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName());
}
private static List<VespaConfigChangeAction> validateDocumentTypeChanges(ClusterSpec.Id id,
@@ -79,8 +72,8 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate();
}
- private static NewDocumentType getDocumentType(ContentCluster cluster, DocumentDatabase db) {
- return cluster.getDocumentDefinitions().get(db.getName());
+ private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) {
+ return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName());
}
private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(ClusterSpec.Id id,
@@ -117,7 +110,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
.toList();
}
- private static List<VespaConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result,
+ private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result,
List<ServiceInfo> services,
String docTypeName) {
return result.stream()
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index c98e2dac1d9..035ae4a06f5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -22,6 +22,7 @@ import com.yahoo.vespa.model.container.component.ContainerSubsystem;
import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import com.yahoo.vespa.model.search.SearchCluster;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import java.util.Collection;
import java.util.HashSet;
@@ -178,7 +179,10 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
}
scB.rankprofiles_configid(sys.getConfigId());
scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName()));
- scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec()));
+ if ( ! (sys instanceof IndexedSearchCluster)) {
+ scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().
+ routespec(((StreamingSearchCluster)sys).getStorageRouteSpec()));
+ }
builder.searchcluster(scB);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index e2fe660cbe9..125d3b10512 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.model.search.NodeSpec;
import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler;
import com.yahoo.vespa.model.search.SearchCluster;
import com.yahoo.vespa.model.search.SearchNode;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import com.yahoo.vespa.model.search.TransactionLogServer;
import com.yahoo.vespa.model.search.Tuning;
import org.w3c.dom.Element;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.LinkedHashMap;
+import java.util.Optional;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Predicate;
@@ -102,12 +104,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog");
var search = new ContentSearchCluster(ancestor, clusterName,
- deployState.getProperties().featureFlags(),
- documentDefinitions,
- globallyDistributedDocuments,
- getFlushOnShutdown(flushOnShutdownElem),
- syncTransactionLog,
- fractionOfMemoryReserved);
+ deployState.getProperties().featureFlags(),
+ documentDefinitions,
+ globallyDistributedDocuments,
+ getFlushOnShutdown(flushOnShutdownElem),
+ syncTransactionLog,
+ fractionOfMemoryReserved);
ModelElement tuning = clusterElem.childByPath("engine.proton.tuning");
if (tuning != null) {
@@ -115,7 +117,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
search.setResourceLimits(resourceLimits);
- buildSearchCluster(deployState, clusterElem, clusterName, search);
+ buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search);
+ buildIndexedSearchCluster(deployState, clusterElem, clusterName, search);
return search;
}
@@ -127,18 +130,73 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
return clusterElem.childAsDouble("engine.proton.query-timeout");
}
- private void buildSearchCluster(DeployState deployState, ModelElement clusterElem,
- String clusterName, ContentSearchCluster search) {
+ private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) {
+ var schemas = deployState.getSchemas();
+ for (var candidate : schemas) {
+ if (candidate.getName().equals(docTypeName)) {
+ return candidate;
+ }
+ }
+ return null;
+ }
+
+ private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) {
ModelElement docElem = clusterElem.child("documents");
- if (docElem == null) return;
- Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
- if (visibilityDelay != null) {
- search.setVisibilityDelay(visibilityDelay);
+ if (docElem == null) {
+ return;
}
- IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
- search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document"));
+ for (ModelElement docType : docElem.subElements("document")) {
+ String docTypeName = docType.stringAttribute("type");
+ String mode = docType.stringAttribute("mode");
+ var schema = findResponsibleSchema(deployState, docTypeName);
+ if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) {
+ buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType);
+ }
+ }
+ }
+
+ private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName,
+ ContentSearchCluster search, ModelElement docType) {
+ String docTypeName = docType.stringAttribute("type");
+ StreamingSearchCluster cluster = new StreamingSearchCluster(search,
+ clusterName + "." + docTypeName,
+ 0,
+ docTypeName,
+ clusterName);
+ search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType));
+ }
+
+ private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem,
+ String clusterName, ContentSearchCluster search) {
+ List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem);
+ if (!indexedDefs.isEmpty()) {
+ IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
+
+ Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
+ if (visibilityDelay != null) {
+ search.setVisibilityDelay(visibilityDelay);
+ }
+
+ search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs);
+ }
+ }
+
+ private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) {
+ List<ModelElement> indexedDefs = new ArrayList<>();
+ ModelElement docElem = clusterElem.child("documents");
+ if (docElem == null) {
+ return indexedDefs;
+ }
+
+ for (ModelElement docType : docElem.subElements("document")) {
+ String mode = docType.stringAttribute("mode");
+ if ("index".equals(mode)) {
+ indexedDefs.add(docType);
+ }
+ }
+ return indexedDefs;
}
}
@@ -189,12 +247,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
cluster.setQueryTimeout(queryTimeout);
}
cluster.deriveFromSchemas(deployState);
- if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) {
- addCluster(cluster);
- }
+ addCluster(cluster);
}
-
private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) {
for (ModelElement e : searchDefs) {
SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e);
@@ -202,9 +257,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
if (schema == null)
throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " +
this + " does not exist");
- if (schema.isDocumentsOnly()) continue;
- sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null));
+ sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null));
}
}
@@ -228,9 +282,14 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
* with indexing, null if it has both or none.
*/
public Boolean isStreaming() {
- if (indexedCluster == null) return false;
- boolean hasStreaming = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING);
- boolean hasIndexed = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.INDEX);
+ boolean hasStreaming = false;
+ boolean hasIndexed = false;
+ for (var cluster : clusters.values()) {
+ if (cluster.isStreaming())
+ hasStreaming = true;
+ else
+ hasIndexed = true;
+ }
if (hasIndexed == hasStreaming) return null;
return hasStreaming;
}
@@ -297,6 +356,21 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
this.redundancy = redundancy;
}
+ private Optional<StreamingSearchCluster> findStreamingCluster(String docType) {
+ return getClusters().values().stream()
+ .filter(StreamingSearchCluster.class::isInstance)
+ .map(StreamingSearchCluster.class::cast)
+ .filter(ssc -> ssc.schemas().get(docType) != null)
+ .findFirst();
+ }
+
+ public List<StreamingSearchCluster> getStreamingClusters() {
+ return getClusters().values().stream()
+ .filter(StreamingSearchCluster.class::isInstance)
+ .map(StreamingSearchCluster.class::cast)
+ .toList();
+ }
+
public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); }
public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); }
public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); }
@@ -308,13 +382,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
private boolean hasIndexingModeStreaming(NewDocumentType type) {
- if (indexedCluster == null) return false;
- return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING;
+ return findStreamingCluster(type.getFullName().getName()).isPresent();
}
private boolean hasIndexingModeIndexed(NewDocumentType type) {
- if (indexedCluster == null) return false;
- return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX;
+ return !hasIndexingModeStreaming(type)
+ && hasIndexedCluster()
+ && getIndexed().hasDocumentDB(type.getFullName().getName());
}
private boolean hasIndexingModeStoreOnly(NewDocumentType type) {
@@ -323,7 +397,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
@Override
public void getConfig(ProtonConfig.Builder builder) {
- boolean hasAnyNonIndexedSchema = false;
+ boolean hasAnyNonIndexedCluster = false;
for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) {
ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
String docTypeName = type.getFullName().getName();
@@ -335,13 +409,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers);
if (hasIndexingModeStreaming(type)) {
- hasAnyNonIndexedSchema = true;
- indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ hasAnyNonIndexedCluster = true;
+ findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING);
} else if (hasIndexingModeIndexed(type)) {
- indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
} else {
- hasAnyNonIndexedSchema = true;
+ hasAnyNonIndexedCluster = true;
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY);
}
if (globalDocType) {
@@ -350,7 +424,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
builder.documentdb(ddbB);
}
- if (hasAnyNonIndexedSchema) {
+ if (hasAnyNonIndexedCluster) {
builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2));
} else {
builder.feeding.concurrency(defaultFeedConcurrency);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 4a37b27d1c7..b51185ddac2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -65,7 +65,7 @@ public class IndexedSearchCluster extends SearchCluster implements
for (SchemaInfo spec : schemas().values()) {
if (spec.fullSchema() instanceof DocumentOnlySchema) continue;
var db = new DocumentDatabase(this, spec.fullSchema().getName(),
- new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()));
+ new DerivedConfiguration(spec.fullSchema(), deployState, false));
add(db);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index 990a8bd25aa..f19af8b62ca 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -8,23 +8,14 @@ import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
-import com.yahoo.vespa.config.search.RankProfilesConfig;
-import com.yahoo.vespa.config.search.SummaryConfig;
-import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
-import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
-import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
-import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
-import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
-import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.config.model.producer.AnyConfigProducer;
import com.yahoo.config.model.producer.TreeConfigProducer;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -44,36 +35,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
private Double queryTimeout;
private Double visibilityDelay = 0.0;
private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>();
- private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>();
- private final List<LegacyStreamingproxy> legacyproxy = new ArrayList<>();
-
- private static class LegacyStreamingproxy extends TreeConfigProducer<AnyConfigProducer> implements
- AttributesConfig.Producer,
- RankProfilesConfig.Producer,
- RankingConstantsConfig.Producer,
- RankingExpressionsConfig.Producer,
- OnnxModelsConfig.Producer,
- JuniperrcConfig.Producer,
- SummaryConfig.Producer,
- VsmsummaryConfig.Producer,
- VsmfieldsConfig.Producer
- {
- private final DocumentDatabase db;
- LegacyStreamingproxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, DocumentDatabase db) {
- super(parent, "cluster." + clusterName + "." + db.getName());
- this.db = new DocumentDatabase(this, db.getName(), db.getDerivedConfiguration());
- }
- @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); }
- @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); }
-
- }
+ private final List<DocumentDatabase> documentDbs = new LinkedList<>();
public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) {
super(parent, "cluster." + clusterName);
@@ -81,28 +43,29 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
this.index = index;
}
- public String getStorageRouteSpec() { return getClusterName(); }
-
public void add(SchemaInfo schema) {
schemas.put(schema.name(), schema);
}
public void add(DocumentDatabase db) {
- if (db.getDerivedConfiguration().isStreaming()) {
- legacyproxy.add(new LegacyStreamingproxy((TreeConfigProducer<AnyConfigProducer>) getParent(), clusterName, db));
- }
- documentDbs.put(db.getName(), db);
+ documentDbs.add(db);
}
public boolean hasDocumentDB(String name) {
- return documentDbs.containsKey(name);
- }
- public DocumentDatabase getDocumentDB(String name) {
- return documentDbs.get(name);
+ for (DocumentDatabase db : documentDbs) {
+ if (db.getName().equals(name)) {
+ return true;
+ }
+ }
+ return false;
}
public String getConfigId(String name) {
- DocumentDatabase db = documentDbs.get(name);
- return db != null ? db.getConfigId() : "";
+ for (DocumentDatabase db : documentDbs) {
+ if (db.getName().equals(name)) {
+ return db.getConfigId();
+ }
+ }
+ return "";
}
/** Returns the schemas that should be active in this cluster. Note: These are added during processing. */
@@ -117,7 +80,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
/** Returns the document databases contained in this cluster */
public List<DocumentDatabase> getDocumentDbs() {
- return documentDbs.values().stream().toList();
+ return Collections.unmodifiableList(documentDbs);
}
public String getClusterName() { return clusterName; }
@@ -134,9 +97,11 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
public final int getClusterIndex() { return index; }
public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) {
- DocumentDatabase db = documentDbs.get(documentType);
- if (db != null) {
- fillDocumentDBConfig(db, builder);
+ for (DocumentDatabase sdoc : documentDbs) {
+ if (sdoc.getName().equals(documentType)) {
+ fillDocumentDBConfig(sdoc, builder);
+ return;
+ }
}
}
@@ -147,7 +112,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
@Override
public void getConfig(DocumentdbInfoConfig.Builder builder) {
- for (DocumentDatabase db : documentDbs.values()) {
+ for (DocumentDatabase db : documentDbs) {
var docDb = new DocumentdbInfoConfig.Documentdb.Builder()
.name(db.getName())
.mode(db.getDerivedConfiguration().isStreaming()
@@ -158,21 +123,21 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
}
@Override
public void getConfig(IndexInfoConfig.Builder builder) {
- new Join(documentDbs.values()).getConfig(builder);
+ new Join(documentDbs).getConfig(builder);
}
@Override
public void getConfig(SchemaInfoConfig.Builder builder) {
- new Join(documentDbs.values()).getConfig(builder);
+ new Join(documentDbs).getConfig(builder);
}
@Override
public void getConfig(IlscriptsConfig.Builder builder) {
- new Join(documentDbs.values()).getConfig(builder);
+ new Join(documentDbs).getConfig(builder);
}
public void getConfig(AttributesConfig.Builder builder) {
- new Join(documentDbs.values()).getConfig(builder);
+ new Join(documentDbs).getConfig(builder);
}
@Override
@@ -202,7 +167,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
* that is handled (by delegating to this) by the {@link IndexedSearchCluster}
* which is the parent to this. This avoids building the config multiple times.
*/
- private record Join(Collection<DocumentDatabase> docDbs) {
+ private record Join(List<DocumentDatabase> docDbs) {
public void getConfig(IndexInfoConfig.Builder builder) {
for (DocumentDatabase docDb : docDbs) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
new file mode 100644
index 00000000000..3f15bc90b8f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -0,0 +1,99 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AnyConfigProducer;
+import com.yahoo.config.model.producer.TreeConfigProducer;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.derived.AttributeFields;
+import com.yahoo.schema.derived.DerivedConfiguration;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
+
+/**
+ * A search cluster of type streaming.
+ *
+ * @author baldersheim
+ * @author vegardh
+ */
+public class StreamingSearchCluster extends SearchCluster implements
+ AttributesConfig.Producer,
+ RankProfilesConfig.Producer,
+ RankingConstantsConfig.Producer,
+ RankingExpressionsConfig.Producer,
+ OnnxModelsConfig.Producer,
+ JuniperrcConfig.Producer,
+ SummaryConfig.Producer,
+ VsmsummaryConfig.Producer,
+ VsmfieldsConfig.Producer
+{
+ private final String storageRouteSpec;
+ private final AttributesProducer attributesConfig;
+ private final String docTypeName;
+
+ public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index,
+ String docTypeName, String storageRouteSpec) {
+ super(parent, clusterName, index);
+ attributesConfig = new AttributesProducer(parent, docTypeName);
+ this.docTypeName = docTypeName;
+ this.storageRouteSpec = storageRouteSpec;
+ }
+
+ @Override
+ protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; }
+ public final String getStorageRouteSpec() { return storageRouteSpec; }
+
+ public String getDocTypeName() { return docTypeName; }
+
+ public DerivedConfiguration derived() { return db().getDerivedConfiguration(); }
+
+ @Override
+ public void deriveFromSchemas(DeployState deployState) {
+ if (schemas().isEmpty()) return;
+ if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size());
+
+ Schema schema = schemas().values().stream().findAny().get().fullSchema();
+ if ( ! schema.getName().equals(docTypeName))
+ throw new IllegalArgumentException("Document type name '" + docTypeName +
+ "' must be the same as the schema name '" + schema.getName() + "'");
+ add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(schema, deployState, true)));
+ }
+
+ protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
+ super.fillDocumentDBConfig(sdoc, ddbB);
+ ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up
+ }
+
+ private DocumentDatabase db() { return getDocumentDbs().get(0); }
+
+ // These are temporary until backend uses correct config id.
+ @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); }
+ @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);}
+
+ private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer {
+
+ AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) {
+ super(parent, docType);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS);
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
index 52ddf238c96..7d7ed2b23eb 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
@@ -47,12 +47,12 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase {
TestProperties properties,
ApplicationBuilder builder,
DeployLogger logger) throws IOException {
- DerivedConfiguration config = new DerivedConfiguration(new DeployState.Builder()
- .properties(properties)
- .deployLogger(logger)
- .rankProfileRegistry(builder.getRankProfileRegistry())
- .queryProfiles(builder.getQueryProfileRegistry())
- .build(), builder.getSchema(schemaName), SchemaInfo.IndexMode.INDEX);
+ DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName),
+ new DeployState.Builder().properties(properties)
+ .deployLogger(logger)
+ .rankProfileRegistry(builder.getRankProfileRegistry())
+ .queryProfiles(builder.getQueryProfileRegistry())
+ .build(), false);
return export(dirName, builder, config);
}
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
index c940afce428..2beab3d5ea9 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
@@ -41,13 +41,12 @@ public class SchemaToDerivedConfigExporter {
var props = new TestProperties();
var logger = new TestableDeployLogger();
var builder = ApplicationBuilder.createFromDirectory(dirPath, new MockFileRegistry(), logger, props);
- var derived = new DerivedConfiguration(
+ var derived = new DerivedConfiguration(builder.getSchema(null),
new DeployState.Builder().properties(props)
.deployLogger(logger)
.rankProfileRegistry(builder.getRankProfileRegistry())
.queryProfiles(builder.getQueryProfileRegistry())
- .build(),
- builder.getSchema(null), SchemaInfo.IndexMode.INDEX);
+ .build(), false);
exportConfig(dirPath, derived, builder);
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
index bdac1b1e268..5397c30f2bc 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
@@ -29,7 +29,7 @@ public class StreamingValidatorTest {
new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/")
.create();
});
- assertTrue(exception.getMessage().contains("For search cluster 'content', schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
+ assertTrue(exception.getMessage().contains("For streaming search cluster 'content.ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
"Document references and imported fields are not allowed in streaming search."));
}
@@ -52,7 +52,7 @@ public class StreamingValidatorTest {
"attribute { distance-metric: euclidean } }");
var warnings = filter(logger.warnings);
assertEquals(1, warnings.size());
- assertEquals("For search cluster 'content', schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
+ assertEquals("For streaming search cluster 'content.test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
warnings.get(0));
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
index 28eabd18539..ee64ceb6969 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
@@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest {
private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }";
private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }";
private static final List<ServiceInfo> FOO_SERVICE =
- List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null"));
+ List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null"));
private static final List<ServiceInfo> BAR_SERVICE =
- List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null"));
+ List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null"));
@Test
void changing_field_type_requires_refeed() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index 327b169a629..f8adb18a2c3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -22,6 +22,7 @@ import com.yahoo.vespa.model.content.engines.ProtonEngine;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import com.yahoo.vespa.model.search.SearchCluster;
import com.yahoo.vespa.model.search.SearchNode;
+import com.yahoo.vespa.model.search.StreamingSearchCluster;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -196,11 +197,12 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s;
s = cluster.getSearch();
- assertTrue(s.hasIndexedCluster());
+ assertFalse(s.hasIndexedCluster());
assertEquals(1, s.getClusters().size());
- SearchCluster sc = s.getClusters().get(musicClusterId);
- assertEquals(musicClusterId, sc.getClusterName());
- assertEquals(musicClusterId, sc.getStorageRouteSpec());
+ assertNull(s.getIndexed());
+ SearchCluster sc = s.getClusters().get(musicClusterId + ".music");
+ assertEquals(musicClusterId + ".music", sc.getClusterName());
+ assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
assertEquals(1, cluster.getStorageCluster().getChildren().size());
@@ -235,11 +237,24 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\"/>" +
" </group>" +
"</content>");
- ContentSearchCluster s = cluster.getSearch();
- assertTrue(s.hasIndexedCluster());
- assertEquals(2, s.getIndexed().getDocumentDbs().size());
- assertTrue(s.getIndexed().hasDocumentDB("book"));
- assertTrue(s.getIndexed().hasDocumentDB("music"));
+ ContentSearchCluster s;
+
+ s = cluster.getSearch();
+ assertFalse(s.hasIndexedCluster());
+ assertEquals(2, s.getClusters().size());
+ assertNull(s.getIndexed());
+ {
+ String id = musicClusterId + ".book";
+ SearchCluster sc = s.getClusters().get(id);
+ assertEquals(id, sc.getClusterName());
+ assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
+ }
+ {
+ String id = musicClusterId + ".music";
+ SearchCluster sc = s.getClusters().get(id);
+ assertEquals(id, sc.getClusterName());
+ assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
+ }
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
assertEquals(1, cluster.getStorageCluster().getChildren().size());
@@ -348,7 +363,9 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
" </group>" +
"</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2")));
- ContentSearchCluster s = b.getSearch();
+ ContentSearchCluster s;
+
+ s = b.getSearch();
assertTrue(s.hasIndexedCluster());
assertNotNull(s.getIndexed());
assertEquals(1, b.getStorageCluster().getChildren().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index ae08f78c404..6f9127779db 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -163,6 +163,7 @@ public class IndexedTest extends ContentBaseTest {
// "transactionlogserver"};
// DomContentBuilderTest.assertServices(h, expectedServices);
ContentCluster s = model.getContentClusters().get("test");
+ assertFalse(s.getSearch().hasIndexedCluster());
StorServerConfig.Builder builder = new StorServerConfig.Builder();
s.getStorageCluster().getConfig(builder);
@@ -174,6 +175,7 @@ public class IndexedTest extends ContentBaseTest {
VespaModel model = getStreamingVespaModel();
ContentCluster s = model.getContentClusters().get("test");
assertNotNull(s);
+ assertFalse(s.getSearch().hasIndexedCluster());
ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID);
assertEquals(1, config.storage().size());
assertEquals("test", config.storage(0).name());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 4cf4068f331..eb4ec1af157 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -1,6 +1,7 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search.test;
+import com.google.common.collect.ImmutableMap;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.path.Path;
@@ -18,6 +19,7 @@ import com.yahoo.vespa.model.content.utils.DocType;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import org.junit.jupiter.api.Test;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -310,7 +312,7 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModelWithMode(mode, sds);
- DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
+ DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
assertEquals(1, dcfg.documentdb().size());
DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0);
assertEquals("type", db.name());
@@ -329,11 +331,13 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModel(sds, "");
DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
- assertEquals(2, indexed_cfg.documentdb().size());
+ assertEquals(1, indexed_cfg.documentdb().size());
var db = indexed_cfg.documentdb(0);
assertEquals("a", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode());
- db = indexed_cfg.documentdb(1);
+ DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b");
+ assertEquals(1, streaming_cfg.documentdb().size());
+ db = streaming_cfg.documentdb(0);
assertEquals("b", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode());
}
@@ -375,7 +379,7 @@ public class DocumentDatabaseTestCase {
void testThatAttributesMaxUnCommittedMemoryIsControlledByFeatureFlag() {
assertAttributesConfigIndependentOfMode("index", List.of("type1"),
List.of("test/search/cluster.test/type1"),
- Map.of("type1", List.of("f2", "f2_nfa")),
+ ImmutableMap.of("type1", List.of("f2", "f2_nfa")),
new DeployState.Builder().properties(new TestProperties().maxUnCommittedMemory(193452)), 193452);
}
@@ -383,20 +387,20 @@ public class DocumentDatabaseTestCase {
void testThatAttributesConfigIsProducedForIndexed() {
assertAttributesConfigIndependentOfMode("index", List.of("type1"),
List.of("test/search/cluster.test/type1"),
- Map.of("type1", List.of("f2", "f2_nfa")));
+ ImmutableMap.of("type1", List.of("f2", "f2_nfa")));
}
@Test
void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() {
assertAttributesConfigIndependentOfMode("streaming", List.of("type1"),
- List.of("test/search/cluster.test/type1"),
- Map.of("type1", List.of("f2")));
+ List.of("test/search/type1"),
+ ImmutableMap.of("type1", List.of("f2")));
}
@Test
void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() {
assertAttributesConfigIndependentOfMode("store-only", List.of("type1"),
- List.of("test/search"), Map.of());
+ List.of("test/search"), Collections.emptyMap());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
index 9e4ca630340..388c064daf4 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
@@ -2,11 +2,13 @@
package com.yahoo.vespa.model.search.test;
import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.document.DataType;
import com.yahoo.schema.RankProfileRegistry;
import com.yahoo.schema.Schema;
import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.derived.Summaries;
import com.yahoo.schema.document.SDDocumentType;
+import com.yahoo.schema.document.SDField;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import com.yahoo.vespa.model.test.utils.DeployLoggerStub;
@@ -25,7 +27,7 @@ public class SchemaInfoTester {
}
public String schemaInfoConfig(Schema schema) {
- var schemaInfo = new SchemaInfo(schema, SchemaInfo.IndexMode.INDEX, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties()));
+ var schemaInfo = new SchemaInfo(schema, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties()));
var schemaInfoConfigBuilder = new SchemaInfoConfig.Builder();
schemaInfo.getConfig(schemaInfoConfigBuilder);
var schemaInfoConfig = schemaInfoConfigBuilder.build();