summaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2024-03-13 10:46:34 +0100
committerHenning Baldersheim <balder@yahoo-inc.com>2024-03-13 16:41:21 +0100
commit4528b9f815cf780fbda7ee0d041b8070a39d518d (patch)
tree4bbc1d7ea32a93b2c4097f6ad997acb2882307eb /config-model
parent37484d712d93d7daf6075455b266f18788d40690 (diff)
Use a single searchcluster with both indexed and streaming docs.
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java23
-rw-r--r--config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java134
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java89
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java3
-rw-r--r--config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java12
-rw-r--r--config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java36
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java20
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java4
21 files changed, 235 insertions, 262 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
index b35918b3460..a7f66966538 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
@@ -28,7 +28,7 @@ import java.io.Writer;
public class DerivedConfiguration {
private final Schema schema;
- private final boolean isStreaming;
+ private final SchemaInfo.IndexMode indexMode;
private Summaries summaries;
private Juniperrc juniperrc;
private AttributeFields attributeFields;
@@ -57,7 +57,8 @@ public class DerivedConfiguration {
}
DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) {
- this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false);
+ this(new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(),
+ schema, SchemaInfo.IndexMode.INDEX);
}
/**
@@ -67,8 +68,8 @@ public class DerivedConfiguration {
* argument is live. Which means that this object will be inconsistent if the given
* schema is later modified.
*/
- public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) {
- this.isStreaming = isStreaming;
+ public DerivedConfiguration(DeployState deployState, Schema schema, SchemaInfo.IndexMode indexMode) {
+ this.indexMode = indexMode;
try {
Validator.ensureNotNull("Schema", schema);
this.schema = schema;
@@ -81,9 +82,9 @@ public class DerivedConfiguration {
summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags());
juniperrc = new Juniperrc(schema);
rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState);
- indexingScript = new IndexingScript(schema, isStreaming);
- indexInfo = new IndexInfo(schema, isStreaming);
- schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries);
+ indexingScript = new IndexingScript(schema, isStreaming());
+ indexInfo = new IndexInfo(schema, isStreaming());
+ schemaInfo = new SchemaInfo(schema, indexMode, deployState.rankProfileRegistry(), summaries);
indexSchema = new IndexSchema(schema);
importedFields = new ImportedFields(schema);
}
@@ -153,7 +154,7 @@ public class DerivedConfiguration {
}
public boolean isStreaming() {
- return isStreaming;
+ return indexMode == SchemaInfo.IndexMode.STREAMING;
}
public Summaries getSummaries() {
@@ -165,7 +166,11 @@ public class DerivedConfiguration {
}
public void getConfig(AttributesConfig.Builder builder) {
- getConfig(builder, AttributeFields.FieldSet.ALL);
+ if (isStreaming()) {
+ getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS);
+ } else {
+ getConfig(builder, AttributeFields.FieldSet.ALL);
+ }
}
public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
index 19a045ac444..f996b2624db 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
@@ -42,13 +42,32 @@ public final class SchemaInfo extends Derived {
private final Map<String, RankProfileInfo> rankProfiles;
private final Summaries summaries;
+ private final IndexMode indexMode;
- public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
+ public enum IndexMode {INDEX, STREAMING, STORE_ONLY}
+
+ public SchemaInfo(Schema schema, String indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
+ this(schema, indexMode(indexMode), rankProfileRegistry, summaries);
+ }
+ public SchemaInfo(Schema schema, IndexMode indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
this.schema = schema;
this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema)));
this.summaries = summaries;
+ this.indexMode = indexMode;
}
+ private static IndexMode indexMode(String mode) {
+ if (mode == null) return IndexMode.INDEX;
+ return switch (mode) {
+ case "index" -> IndexMode.INDEX;
+ case "streaming" -> IndexMode.STREAMING;
+ case "store-only" -> IndexMode.STORE_ONLY;
+ default -> IndexMode.STORE_ONLY;
+ };
+ }
+
+ public IndexMode getIndexMode() { return indexMode; }
+
public String name() { return schema.getName(); }
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
index 12f3f025996..7b7c92a43e0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
@@ -28,10 +28,10 @@ public class ComplexFieldsWithStructFieldAttributesValidator implements Validato
public void validate(Context context) {
List<SearchCluster> searchClusters = context.model().getSearchClusters();
for (SearchCluster cluster : searchClusters) {
- if (cluster.isStreaming()) continue;
-
for (SchemaInfo spec : cluster.schemas().values()) {
- validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger());
+ if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
+ validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger());
+ }
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
index 3dbba081400..2d553c84f48 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.model.application.validation;
import com.yahoo.schema.Schema;
+import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.vespa.model.application.validation.Validation.Context;
@@ -24,11 +25,10 @@ public class ComplexFieldsWithStructFieldIndexesValidator implements Validator {
@Override
public void validate(Context context) {
for (var cluster : context.model().getSearchClusters()) {
- if (cluster.isStreaming()) {
- continue;
- }
for (var spec : cluster.schemas().values()) {
- validateComplexFields(context, cluster.getClusterName(), spec.fullSchema());
+ if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
+ validateComplexFields(context, cluster.getClusterName(), spec.fullSchema());
+ }
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
index 15d293e4abc..e1c64422653 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
@@ -23,10 +23,9 @@ public class NoPrefixForIndexes implements Validator {
@Override
public void validate(Context context) {
for (SearchCluster cluster : context.model().getSearchClusters()) {
- if (cluster instanceof IndexedSearchCluster) {
- IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
- for (DocumentDatabase docDb : sc.getDocumentDbs()) {
- DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
+ for (DocumentDatabase docDb : cluster.getDocumentDbs()) {
+ DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
+ if ( ! sdConfig.isStreaming() ) {
Schema schema = sdConfig.getSchema();
for (ImmutableSDField field : schema.allConcreteFields()) {
if (field.doesIndexing()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 03076b64d0c..736665b2184 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -69,7 +69,7 @@ public class RankSetupValidator implements Validator {
String schemaDir = clusterDir + schemaName + "/";
writeConfigs(schemaDir, docDb);
writeExtraVerifyRankSetupConfig(schemaDir, docDb);
- if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, sc.isStreaming())) {
+ if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, docDb.getDerivedConfiguration().isStreaming())) {
return;
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index 6e21adc4fe4..cd8094f8c12 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -31,15 +31,11 @@ public class SearchDataTypeValidator implements Validator {
public void validate(Context context) {
List<SearchCluster> clusters = context.model().getSearchClusters();
for (SearchCluster cluster : clusters) {
- if (cluster.isStreaming()) {
- continue;
- }
for (SchemaInfo spec : cluster.schemas().values()) {
SDDocumentType docType = spec.fullSchema().getDocument();
- if (docType == null) {
- continue;
+ if (docType != null && spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) {
+ validateDocument(context, cluster, spec.fullSchema(), docType);
}
- validateDocument(context, cluster, spec.fullSchema(), docType);
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
index c0ad55fc8f4..0e9d3bf7016 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
@@ -6,12 +6,14 @@ import com.yahoo.document.DataType;
import com.yahoo.document.NumericDataType;
import com.yahoo.document.TensorDataType;
import com.yahoo.documentmodel.NewDocumentReferenceDataType;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.derived.DerivedConfiguration;
+import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.schema.document.MatchType;
import com.yahoo.vespa.model.application.validation.Validation.Context;
import com.yahoo.vespa.model.search.SearchCluster;
-import com.yahoo.vespa.model.search.StreamingSearchCluster;
import java.util.List;
import java.util.logging.Level;
@@ -25,20 +27,21 @@ public class StreamingValidator implements Validator {
public void validate(Context context) {
List<SearchCluster> searchClusters = context.model().getSearchClusters();
for (SearchCluster cluster : searchClusters) {
- if ( ! cluster.isStreaming()) continue;
-
- var streamingCluster = (StreamingSearchCluster)cluster;
- warnStreamingAttributes(streamingCluster, context.deployState().getDeployLogger());
- warnStreamingGramMatching(streamingCluster, context.deployState().getDeployLogger());
- failStreamingDocumentReferences(context, streamingCluster);
+ for (SchemaInfo schemaInfo : cluster.schemas().values()) {
+ if (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING) {
+ var deployLogger = context.deployState().getDeployLogger();
+ warnStreamingAttributes(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger);
+ warnStreamingGramMatching(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger);
+ failStreamingDocumentReferences(cluster.getClusterName(), cluster.getDocumentDB(schemaInfo.name()).getDerivedConfiguration(), context);
+ }
+ }
}
}
- private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) {
- for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) {
+ private static void warnStreamingGramMatching(String cluster, Schema schema, DeployLogger logger) {
+ for (ImmutableSDField sd : schema.allConcreteFields()) {
if (sd.getMatching().getType() == MatchType.GRAM) {
- logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" +
- sc.getClusterName() +
+ logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + "', schema '" + schema.getName() +
"', SD field '" + sd.getName() +
"': n-gram matching is not supported for streaming search.");
}
@@ -47,19 +50,16 @@ public class StreamingValidator implements Validator {
/**
* Warn if one or more attributes are defined in a streaming search cluster SD.
- *
- * @param sc a search cluster to be checked for attributes in streaming search
- * @param logger a DeployLogger
*/
- private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) {
- for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) {
+ private static void warnStreamingAttributes(String cluster, Schema schema, DeployLogger logger) {
+ for (ImmutableSDField sd : schema.allConcreteFields()) {
if (sd.doesAttributing()) {
- warnStreamingAttribute(sc, sd, logger);
+ warnStreamingAttribute(cluster, schema.getName(), sd, logger);
}
}
}
- private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) {
+ private static void warnStreamingAttribute(String cluster, String schema, ImmutableSDField sd, DeployLogger logger) {
// If the field is numeric, we can't print this, because we may have converted the field to
// attribute indexing ourselves (IntegerIndex2Attribute)
if (sd.getDataType() instanceof NumericDataType) return;
@@ -68,25 +68,25 @@ public class StreamingValidator implements Validator {
for (var fieldAttribute : sd.getAttributes().values()) {
if (fieldAttribute.hnswIndexParams().isPresent()) {
logger.logApplicationPackage(Level.WARNING,
- "For streaming search cluster '" + sc.getClusterName() +
+ "For search cluster '" + cluster + "', schema '" + schema +
"', SD field '" + sd.getName() +
"': hnsw index is not relevant and not supported, ignoring setting");
}
}
return;
}
- logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() +
+ logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster +
"', SD field '" + sd.getName() +
"': 'attribute' has same match semantics as 'index'.");
}
- private static void failStreamingDocumentReferences(Context context, StreamingSearchCluster sc) {
- for (Attribute attribute : sc.derived().getAttributeFields().attributes()) {
+ private static void failStreamingDocumentReferences(String cluster, DerivedConfiguration derived, Context context) {
+ for (Attribute attribute : derived.getAttributeFields().attributes()) {
DataType dataType = attribute.getDataType();
if (dataType instanceof NewDocumentReferenceDataType) {
- String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " +
+ String errorMessage = String.format("For search cluster '%s', schema '%s': Attribute '%s' has type '%s'. " +
"Document references and imported fields are not allowed in streaming search.",
- sc.getClusterName(), attribute.getName(), dataType.getName());
+ cluster, derived.getSchema().getName(), attribute.getName(), dataType.getName());
context.illegal(errorMessage);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
index 3b89467299d..0d42219dade 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java
@@ -1,7 +1,6 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.application.validation.change;
-import com.yahoo.config.model.api.ConfigChangeAction;
import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.documentmodel.NewDocumentType;
@@ -12,11 +11,11 @@ import com.yahoo.vespa.model.application.validation.Validation.ChangeContext;
import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder;
import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
-import com.yahoo.vespa.model.search.StreamingSearchCluster;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+import com.yahoo.vespa.model.search.SearchCluster;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -31,39 +30,47 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> {
ContentCluster nextCluster = context.model().getContentClusters().get(clusterName);
if (nextCluster != null) {
- List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters();
- currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> {
- Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters);
- nextStreamingCluster.ifPresent(streamingSearchCluster -> validateStreamingCluster(currentCluster, currentStreamingCluster,
- nextCluster, streamingSearchCluster).forEach(context::require));
+ var nextStreamingClusters = nextCluster.getSearch().getClusters();
+ currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> {
+ SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName());
+ if (nextStreamingCluster != null) {
+ validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require);
+ }
});
}
});
}
- private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) {
- return clusters.stream()
- .filter(cluster -> cluster.getClusterName().equals(clusterName))
- .findFirst();
+ private static List<VespaConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster,
+ SearchCluster currentStreamingCluster,
+ ContentCluster nextCluster,
+ SearchCluster nextStreamingCluster) {
+ List<VespaConfigChangeAction> result = new ArrayList<>();
+
+ for (DocumentDatabase currentDB : currentStreamingCluster.getDocumentDbs()) {
+ DocumentDatabase nextDB = nextStreamingCluster.getDocumentDB(currentDB.getName());
+ if (nextDB != null) {
+ result.addAll(validateDocumentDB(currentCluster, currentDB, nextCluster, nextDB));
+ }
+ }
+ return result;
}
- private static List<ConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster,
- StreamingSearchCluster currentStreamingCluster,
- ContentCluster nextCluster,
- StreamingSearchCluster nextStreamingCluster) {
+ private static List<VespaConfigChangeAction> validateDocumentDB(ContentCluster currentCluster, DocumentDatabase currentDB,
+ ContentCluster nextCluster, DocumentDatabase nextDB) {
List<VespaConfigChangeAction> result = new ArrayList<>();
result.addAll(validateDocumentTypeChanges(currentCluster.id(),
- getDocumentType(currentCluster, currentStreamingCluster),
- getDocumentType(nextCluster, nextStreamingCluster)));
+ getDocumentType(currentCluster, currentDB),
+ getDocumentType(nextCluster, nextDB)));
result.addAll(validateAttributeFastAccessAdded(currentCluster.id(),
- currentStreamingCluster.derived().getAttributeFields(),
- nextStreamingCluster.derived().getAttributeFields()));
+ currentDB.getDerivedConfiguration().getAttributeFields(),
+ nextDB.getDerivedConfiguration().getAttributeFields()));
result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(),
- currentStreamingCluster.derived().getAttributeFields(),
- nextStreamingCluster.derived().getAttributeFields()));
+ currentDB.getDerivedConfiguration().getAttributeFields(),
+ nextDB.getDerivedConfiguration().getAttributeFields()));
- return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName());
+ return modifyActions(result, getSearchNodeServices(nextCluster), nextDB.getName());
}
private static List<VespaConfigChangeAction> validateDocumentTypeChanges(ClusterSpec.Id id,
@@ -72,8 +79,8 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate();
}
- private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) {
- return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName());
+ private static NewDocumentType getDocumentType(ContentCluster cluster, DocumentDatabase db) {
+ return cluster.getDocumentDefinitions().get(db.getName());
}
private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(ClusterSpec.Id id,
@@ -110,7 +117,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator {
.toList();
}
- private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result,
+ private static List<VespaConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result,
List<ServiceInfo> services,
String docTypeName) {
return result.stream()
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 125d3b10512..9d2a7e722ff 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -31,7 +31,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.LinkedHashMap;
-import java.util.Optional;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Predicate;
@@ -104,12 +103,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog");
var search = new ContentSearchCluster(ancestor, clusterName,
- deployState.getProperties().featureFlags(),
- documentDefinitions,
- globallyDistributedDocuments,
- getFlushOnShutdown(flushOnShutdownElem),
- syncTransactionLog,
- fractionOfMemoryReserved);
+ deployState.getProperties().featureFlags(),
+ documentDefinitions,
+ globallyDistributedDocuments,
+ getFlushOnShutdown(flushOnShutdownElem),
+ syncTransactionLog,
+ fractionOfMemoryReserved);
ModelElement tuning = clusterElem.childByPath("engine.proton.tuning");
if (tuning != null) {
@@ -117,8 +116,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
search.setResourceLimits(resourceLimits);
- buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search);
- buildIndexedSearchCluster(deployState, clusterElem, clusterName, search);
+ buildSearchCluster(deployState, clusterElem, clusterName, search);
return search;
}
@@ -130,73 +128,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
return clusterElem.childAsDouble("engine.proton.query-timeout");
}
- private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) {
- var schemas = deployState.getSchemas();
- for (var candidate : schemas) {
- if (candidate.getName().equals(docTypeName)) {
- return candidate;
- }
- }
- return null;
- }
-
- private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) {
+ private void buildSearchCluster(DeployState deployState, ModelElement clusterElem,
+ String clusterName, ContentSearchCluster search) {
ModelElement docElem = clusterElem.child("documents");
+ if (docElem == null) return;
- if (docElem == null) {
- return;
+ Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
+ if (visibilityDelay != null) {
+ search.setVisibilityDelay(visibilityDelay);
}
- for (ModelElement docType : docElem.subElements("document")) {
- String docTypeName = docType.stringAttribute("type");
- String mode = docType.stringAttribute("mode");
- var schema = findResponsibleSchema(deployState, docTypeName);
- if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) {
- buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType);
- }
- }
- }
-
- private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName,
- ContentSearchCluster search, ModelElement docType) {
- String docTypeName = docType.stringAttribute("type");
- StreamingSearchCluster cluster = new StreamingSearchCluster(search,
- clusterName + "." + docTypeName,
- 0,
- docTypeName,
- clusterName);
- search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType));
- }
-
- private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem,
- String clusterName, ContentSearchCluster search) {
- List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem);
- if (!indexedDefs.isEmpty()) {
- IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
-
- Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
- if (visibilityDelay != null) {
- search.setVisibilityDelay(visibilityDelay);
- }
-
- search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs);
- }
- }
-
- private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) {
- List<ModelElement> indexedDefs = new ArrayList<>();
- ModelElement docElem = clusterElem.child("documents");
- if (docElem == null) {
- return indexedDefs;
- }
-
- for (ModelElement docType : docElem.subElements("document")) {
- String mode = docType.stringAttribute("mode");
- if ("index".equals(mode)) {
- indexedDefs.add(docType);
- }
- }
- return indexedDefs;
+ IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags());
+ search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document"));
}
}
@@ -247,9 +190,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
cluster.setQueryTimeout(queryTimeout);
}
cluster.deriveFromSchemas(deployState);
- addCluster(cluster);
+ if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) {
+ addCluster(cluster);
+ }
}
+
private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) {
for (ModelElement e : searchDefs) {
SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e);
@@ -257,8 +203,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
if (schema == null)
throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " +
this + " does not exist");
+ if (schema.isDocumentsOnly()) continue;
- sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null));
+ sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null));
}
}
@@ -282,14 +229,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
* with indexing, null if it has both or none.
*/
public Boolean isStreaming() {
- boolean hasStreaming = false;
- boolean hasIndexed = false;
- for (var cluster : clusters.values()) {
- if (cluster.isStreaming())
- hasStreaming = true;
- else
- hasIndexed = true;
- }
+ if (indexedCluster == null) return false;
+ boolean hasStreaming = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING);
+ boolean hasIndexed = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.INDEX);
if (hasIndexed == hasStreaming) return null;
return hasStreaming;
}
@@ -356,14 +298,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
this.redundancy = redundancy;
}
- private Optional<StreamingSearchCluster> findStreamingCluster(String docType) {
- return getClusters().values().stream()
- .filter(StreamingSearchCluster.class::isInstance)
- .map(StreamingSearchCluster.class::cast)
- .filter(ssc -> ssc.schemas().get(docType) != null)
- .findFirst();
- }
-
public List<StreamingSearchCluster> getStreamingClusters() {
return getClusters().values().stream()
.filter(StreamingSearchCluster.class::isInstance)
@@ -382,13 +316,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
}
private boolean hasIndexingModeStreaming(NewDocumentType type) {
- return findStreamingCluster(type.getFullName().getName()).isPresent();
+ if (indexedCluster == null) return false;
+ return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING;
}
private boolean hasIndexingModeIndexed(NewDocumentType type) {
- return !hasIndexingModeStreaming(type)
- && hasIndexedCluster()
- && getIndexed().hasDocumentDB(type.getFullName().getName());
+ if (indexedCluster == null) return false;
+ return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX;
}
private boolean hasIndexingModeStoreOnly(NewDocumentType type) {
@@ -397,7 +331,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
@Override
public void getConfig(ProtonConfig.Builder builder) {
- boolean hasAnyNonIndexedCluster = false;
+ boolean hasAnyNonIndexedSchema = false;
for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) {
ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
String docTypeName = type.getFullName().getName();
@@ -409,13 +343,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers);
if (hasIndexingModeStreaming(type)) {
- hasAnyNonIndexedCluster = true;
- findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ hasAnyNonIndexedSchema = true;
+ indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING);
} else if (hasIndexingModeIndexed(type)) {
- getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB);
} else {
- hasAnyNonIndexedCluster = true;
+ hasAnyNonIndexedSchema = true;
ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY);
}
if (globalDocType) {
@@ -424,7 +358,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
builder.documentdb(ddbB);
}
- if (hasAnyNonIndexedCluster) {
+ if (hasAnyNonIndexedSchema) {
builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2));
} else {
builder.feeding.concurrency(defaultFeedConcurrency);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index b51185ddac2..4a37b27d1c7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -65,7 +65,7 @@ public class IndexedSearchCluster extends SearchCluster implements
for (SchemaInfo spec : schemas().values()) {
if (spec.fullSchema() instanceof DocumentOnlySchema) continue;
var db = new DocumentDatabase(this, spec.fullSchema().getName(),
- new DerivedConfiguration(spec.fullSchema(), deployState, false));
+ new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()));
add(db);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index f19af8b62ca..990a8bd25aa 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -8,14 +8,23 @@ import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.config.model.producer.AnyConfigProducer;
import com.yahoo.config.model.producer.TreeConfigProducer;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -35,7 +44,36 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
private Double queryTimeout;
private Double visibilityDelay = 0.0;
private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>();
- private final List<DocumentDatabase> documentDbs = new LinkedList<>();
+ private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>();
+ private final List<LegacyStreamingproxy> legacyproxy = new ArrayList<>();
+
+ private static class LegacyStreamingproxy extends TreeConfigProducer<AnyConfigProducer> implements
+ AttributesConfig.Producer,
+ RankProfilesConfig.Producer,
+ RankingConstantsConfig.Producer,
+ RankingExpressionsConfig.Producer,
+ OnnxModelsConfig.Producer,
+ JuniperrcConfig.Producer,
+ SummaryConfig.Producer,
+ VsmsummaryConfig.Producer,
+ VsmfieldsConfig.Producer
+ {
+ private final DocumentDatabase db;
+ LegacyStreamingproxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, DocumentDatabase db) {
+ super(parent, "cluster." + clusterName + "." + db.getName());
+ this.db = new DocumentDatabase(this, db.getName(), db.getDerivedConfiguration());
+ }
+ @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); }
+ @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); }
+
+ }
public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) {
super(parent, "cluster." + clusterName);
@@ -43,29 +81,28 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
this.index = index;
}
+ public String getStorageRouteSpec() { return getClusterName(); }
+
public void add(SchemaInfo schema) {
schemas.put(schema.name(), schema);
}
public void add(DocumentDatabase db) {
- documentDbs.add(db);
+ if (db.getDerivedConfiguration().isStreaming()) {
+ legacyproxy.add(new LegacyStreamingproxy((TreeConfigProducer<AnyConfigProducer>) getParent(), clusterName, db));
+ }
+ documentDbs.put(db.getName(), db);
}
public boolean hasDocumentDB(String name) {
- for (DocumentDatabase db : documentDbs) {
- if (db.getName().equals(name)) {
- return true;
- }
- }
- return false;
+ return documentDbs.containsKey(name);
+ }
+ public DocumentDatabase getDocumentDB(String name) {
+ return documentDbs.get(name);
}
public String getConfigId(String name) {
- for (DocumentDatabase db : documentDbs) {
- if (db.getName().equals(name)) {
- return db.getConfigId();
- }
- }
- return "";
+ DocumentDatabase db = documentDbs.get(name);
+ return db != null ? db.getConfigId() : "";
}
/** Returns the schemas that should be active in this cluster. Note: These are added during processing. */
@@ -80,7 +117,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
/** Returns the document databases contained in this cluster */
public List<DocumentDatabase> getDocumentDbs() {
- return Collections.unmodifiableList(documentDbs);
+ return documentDbs.values().stream().toList();
}
public String getClusterName() { return clusterName; }
@@ -97,11 +134,9 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
public final int getClusterIndex() { return index; }
public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) {
- for (DocumentDatabase sdoc : documentDbs) {
- if (sdoc.getName().equals(documentType)) {
- fillDocumentDBConfig(sdoc, builder);
- return;
- }
+ DocumentDatabase db = documentDbs.get(documentType);
+ if (db != null) {
+ fillDocumentDBConfig(db, builder);
}
}
@@ -112,7 +147,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
@Override
public void getConfig(DocumentdbInfoConfig.Builder builder) {
- for (DocumentDatabase db : documentDbs) {
+ for (DocumentDatabase db : documentDbs.values()) {
var docDb = new DocumentdbInfoConfig.Documentdb.Builder()
.name(db.getName())
.mode(db.getDerivedConfiguration().isStreaming()
@@ -123,21 +158,21 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
}
@Override
public void getConfig(IndexInfoConfig.Builder builder) {
- new Join(documentDbs).getConfig(builder);
+ new Join(documentDbs.values()).getConfig(builder);
}
@Override
public void getConfig(SchemaInfoConfig.Builder builder) {
- new Join(documentDbs).getConfig(builder);
+ new Join(documentDbs.values()).getConfig(builder);
}
@Override
public void getConfig(IlscriptsConfig.Builder builder) {
- new Join(documentDbs).getConfig(builder);
+ new Join(documentDbs.values()).getConfig(builder);
}
public void getConfig(AttributesConfig.Builder builder) {
- new Join(documentDbs).getConfig(builder);
+ new Join(documentDbs.values()).getConfig(builder);
}
@Override
@@ -167,7 +202,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer
* that is handled (by delegating to this) by the {@link IndexedSearchCluster}
* which is the parent to this. This avoids building the config multiple times.
*/
- private record Join(List<DocumentDatabase> docDbs) {
+ private record Join(Collection<DocumentDatabase> docDbs) {
public void getConfig(IndexInfoConfig.Builder builder) {
for (DocumentDatabase docDb : docDbs) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index 3f15bc90b8f..e188a086614 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -7,6 +7,7 @@ import com.yahoo.config.model.producer.TreeConfigProducer;
import com.yahoo.schema.Schema;
import com.yahoo.schema.derived.AttributeFields;
import com.yahoo.schema.derived.DerivedConfiguration;
+import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.SummaryConfig;
@@ -64,7 +65,7 @@ public class StreamingSearchCluster extends SearchCluster implements
if ( ! schema.getName().equals(docTypeName))
throw new IllegalArgumentException("Document type name '" + docTypeName +
"' must be the same as the schema name '" + schema.getName() + "'");
- add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(schema, deployState, true)));
+ add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING)));
}
protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
index 7d7ed2b23eb..52ddf238c96 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
@@ -47,12 +47,12 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase {
TestProperties properties,
ApplicationBuilder builder,
DeployLogger logger) throws IOException {
- DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName),
- new DeployState.Builder().properties(properties)
- .deployLogger(logger)
- .rankProfileRegistry(builder.getRankProfileRegistry())
- .queryProfiles(builder.getQueryProfileRegistry())
- .build(), false);
+ DerivedConfiguration config = new DerivedConfiguration(new DeployState.Builder()
+ .properties(properties)
+ .deployLogger(logger)
+ .rankProfileRegistry(builder.getRankProfileRegistry())
+ .queryProfiles(builder.getQueryProfileRegistry())
+ .build(), builder.getSchema(schemaName), SchemaInfo.IndexMode.INDEX);
return export(dirName, builder, config);
}
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
index 2beab3d5ea9..c940afce428 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
@@ -41,12 +41,13 @@ public class SchemaToDerivedConfigExporter {
var props = new TestProperties();
var logger = new TestableDeployLogger();
var builder = ApplicationBuilder.createFromDirectory(dirPath, new MockFileRegistry(), logger, props);
- var derived = new DerivedConfiguration(builder.getSchema(null),
+ var derived = new DerivedConfiguration(
new DeployState.Builder().properties(props)
.deployLogger(logger)
.rankProfileRegistry(builder.getRankProfileRegistry())
.queryProfiles(builder.getQueryProfileRegistry())
- .build(), false);
+ .build(),
+ builder.getSchema(null), SchemaInfo.IndexMode.INDEX);
exportConfig(dirPath, derived, builder);
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
index 5397c30f2bc..bdac1b1e268 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java
@@ -29,7 +29,7 @@ public class StreamingValidatorTest {
new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/")
.create();
});
- assertTrue(exception.getMessage().contains("For streaming search cluster 'content.ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
+ assertTrue(exception.getMessage().contains("For search cluster 'content', schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " +
"Document references and imported fields are not allowed in streaming search."));
}
@@ -52,7 +52,7 @@ public class StreamingValidatorTest {
"attribute { distance-metric: euclidean } }");
var warnings = filter(logger.warnings);
assertEquals(1, warnings.size());
- assertEquals("For streaming search cluster 'content.test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
+ assertEquals("For search cluster 'content', schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting",
warnings.get(0));
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
index ee64ceb6969..28eabd18539 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java
@@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest {
private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }";
private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }";
private static final List<ServiceInfo> FOO_SERVICE =
- List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null"));
+ List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null"));
private static final List<ServiceInfo> BAR_SERVICE =
- List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null"));
+ List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null"));
@Test
void changing_field_type_requires_refeed() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index f8adb18a2c3..0fd535816b1 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -197,12 +197,11 @@ public class ContentBuilderTest extends DomBuilderTest {
ContentSearchCluster s;
s = cluster.getSearch();
- assertFalse(s.hasIndexedCluster());
+ assertTrue(s.hasIndexedCluster());
assertEquals(1, s.getClusters().size());
- assertNull(s.getIndexed());
- SearchCluster sc = s.getClusters().get(musicClusterId + ".music");
- assertEquals(musicClusterId + ".music", sc.getClusterName());
- assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
+ SearchCluster sc = s.getClusters().get(musicClusterId);
+ assertEquals(musicClusterId, sc.getClusterName());
+ assertEquals(musicClusterId, sc.getStorageRouteSpec());
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
assertEquals(1, cluster.getStorageCluster().getChildren().size());
@@ -237,24 +236,11 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\"/>" +
" </group>" +
"</content>");
- ContentSearchCluster s;
-
- s = cluster.getSearch();
- assertFalse(s.hasIndexedCluster());
- assertEquals(2, s.getClusters().size());
- assertNull(s.getIndexed());
- {
- String id = musicClusterId + ".book";
- SearchCluster sc = s.getClusters().get(id);
- assertEquals(id, sc.getClusterName());
- assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
- }
- {
- String id = musicClusterId + ".music";
- SearchCluster sc = s.getClusters().get(id);
- assertEquals(id, sc.getClusterName());
- assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec());
- }
+ ContentSearchCluster s = cluster.getSearch();
+ assertTrue(s.hasIndexedCluster());
+ assertEquals(2, s.getIndexed().getDocumentDbs().size());
+ assertTrue(s.getIndexed().hasDocumentDB("book"));
+ assertTrue(s.getIndexed().hasDocumentDB("music"));
assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory);
assertEquals(1, cluster.getStorageCluster().getChildren().size());
@@ -363,9 +349,7 @@ public class ContentBuilderTest extends DomBuilderTest {
" <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
" </group>" +
"</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2")));
- ContentSearchCluster s;
-
- s = b.getSearch();
+ ContentSearchCluster s = b.getSearch();
assertTrue(s.hasIndexedCluster());
assertNotNull(s.getIndexed());
assertEquals(1, b.getStorageCluster().getChildren().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index 6f9127779db..ae08f78c404 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -163,7 +163,6 @@ public class IndexedTest extends ContentBaseTest {
// "transactionlogserver"};
// DomContentBuilderTest.assertServices(h, expectedServices);
ContentCluster s = model.getContentClusters().get("test");
- assertFalse(s.getSearch().hasIndexedCluster());
StorServerConfig.Builder builder = new StorServerConfig.Builder();
s.getStorageCluster().getConfig(builder);
@@ -175,7 +174,6 @@ public class IndexedTest extends ContentBaseTest {
VespaModel model = getStreamingVespaModel();
ContentCluster s = model.getContentClusters().get("test");
assertNotNull(s);
- assertFalse(s.getSearch().hasIndexedCluster());
ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID);
assertEquals(1, config.storage().size());
assertEquals("test", config.storage(0).name());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index eb4ec1af157..4cf4068f331 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -1,7 +1,6 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search.test;
-import com.google.common.collect.ImmutableMap;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.path.Path;
@@ -19,7 +18,6 @@ import com.yahoo.vespa.model.content.utils.DocType;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import org.junit.jupiter.api.Test;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -312,7 +310,7 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModelWithMode(mode, sds);
- DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
+ DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
assertEquals(1, dcfg.documentdb().size());
DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0);
assertEquals("type", db.name());
@@ -331,13 +329,11 @@ public class DocumentDatabaseTestCase {
var tester = new SchemaTester();
var model = tester.createModel(sds, "");
DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test");
- assertEquals(1, indexed_cfg.documentdb().size());
+ assertEquals(2, indexed_cfg.documentdb().size());
var db = indexed_cfg.documentdb(0);
assertEquals("a", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode());
- DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b");
- assertEquals(1, streaming_cfg.documentdb().size());
- db = streaming_cfg.documentdb(0);
+ db = indexed_cfg.documentdb(1);
assertEquals("b", db.name());
assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode());
}
@@ -379,7 +375,7 @@ public class DocumentDatabaseTestCase {
void testThatAttributesMaxUnCommittedMemoryIsControlledByFeatureFlag() {
assertAttributesConfigIndependentOfMode("index", List.of("type1"),
List.of("test/search/cluster.test/type1"),
- ImmutableMap.of("type1", List.of("f2", "f2_nfa")),
+ Map.of("type1", List.of("f2", "f2_nfa")),
new DeployState.Builder().properties(new TestProperties().maxUnCommittedMemory(193452)), 193452);
}
@@ -387,20 +383,20 @@ public class DocumentDatabaseTestCase {
void testThatAttributesConfigIsProducedForIndexed() {
assertAttributesConfigIndependentOfMode("index", List.of("type1"),
List.of("test/search/cluster.test/type1"),
- ImmutableMap.of("type1", List.of("f2", "f2_nfa")));
+ Map.of("type1", List.of("f2", "f2_nfa")));
}
@Test
void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() {
assertAttributesConfigIndependentOfMode("streaming", List.of("type1"),
- List.of("test/search/type1"),
- ImmutableMap.of("type1", List.of("f2")));
+ List.of("test/search/cluster.test/type1"),
+ Map.of("type1", List.of("f2")));
}
@Test
void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() {
assertAttributesConfigIndependentOfMode("store-only", List.of("type1"),
- List.of("test/search"), Collections.emptyMap());
+ List.of("test/search"), Map.of());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
index 388c064daf4..9e4ca630340 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java
@@ -2,13 +2,11 @@
package com.yahoo.vespa.model.search.test;
import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.document.DataType;
import com.yahoo.schema.RankProfileRegistry;
import com.yahoo.schema.Schema;
import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.schema.derived.Summaries;
import com.yahoo.schema.document.SDDocumentType;
-import com.yahoo.schema.document.SDField;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import com.yahoo.vespa.model.test.utils.DeployLoggerStub;
@@ -27,7 +25,7 @@ public class SchemaInfoTester {
}
public String schemaInfoConfig(Schema schema) {
- var schemaInfo = new SchemaInfo(schema, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties()));
+ var schemaInfo = new SchemaInfo(schema, SchemaInfo.IndexMode.INDEX, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties()));
var schemaInfoConfigBuilder = new SchemaInfoConfig.Builder();
schemaInfo.getConfig(schemaInfoConfigBuilder);
var schemaInfoConfig = schemaInfoConfigBuilder.build();