diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/vespa/model/content')
4 files changed, 76 insertions, 147 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 1254f8e110a..3ccc44fe60b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -135,8 +135,12 @@ public class Content extends ConfigModel { public static List<SearchCluster> getSearchClusters(ConfigModelRepo pc) { List<SearchCluster> clusters = new ArrayList<>(); - for (ContentCluster c : getContentClusters(pc)) - clusters.addAll(c.getSearch().getClusters().values()); + for (ContentCluster c : getContentClusters(pc)) { + SearchCluster sc = c.getSearch().getSearchCluster(); + if (sc != null) { + clusters.add(sc); + } + } return clusters; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a92362bf899..ebb556a2ba2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -31,9 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.Objects; -import java.util.TreeMap; import java.util.function.Predicate; /** @@ -53,11 +50,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> private final boolean flushOnShutdown; private final Boolean syncTransactionLog; - /** If this is set up for streaming search, it is modelled as one search cluster per search definition */ - private final Map<String, SearchCluster> clusters = new TreeMap<>(); - /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ - private IndexedSearchCluster indexedCluster; + private IndexedSearchCluster searchCluster; private final IndexingDocproc indexingDocproc; private Redundancy redundancy; @@ -115,8 +109,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); - buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); + buildSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -128,73 +121,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { - var schemas = deployState.getSchemas(); - for (var candidate : schemas) { - if (candidate.getName().equals(docTypeName)) { - return candidate; - } - } - return null; - } - - private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) return; - if (docElem == null) { - return; - } - - for (ModelElement docType : docElem.subElements("document")) { - String docTypeName = docType.stringAttribute("type"); - String mode = docType.stringAttribute("mode"); - var schema = findResponsibleSchema(deployState, docTypeName); - if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { - buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); - } - } - } - - private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, - ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.stringAttribute("type"); - StreamingSearchCluster cluster = new StreamingSearchCluster(search, - clusterName + "." + docTypeName, - 0, - docTypeName, - clusterName); - search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); - } - - private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); - if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); - } - - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); - } - } - - private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { - List<ModelElement> indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) { - return indexedDefs; + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); } - for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.stringAttribute("mode"); - if ("index".equals(mode)) { - indexedDefs.add(docType); - } - } - return indexedDefs; + var isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); } } @@ -233,21 +171,24 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public void setVisibilityDelay(double delay) { this.visibilityDelay=delay; - if (hasIndexedCluster()) { - indexedCluster.setVisibilityDelay(delay); + if (searchCluster != null) { + searchCluster.setVisibilityDelay(delay); } } - private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { + private void addSearchCluster(DeployState deployState, IndexedSearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { addSchemas(deployState, documentDefs, cluster); if (queryTimeout != null) { cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - addCluster(cluster); + if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { + addCluster(cluster); + } } + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -255,22 +196,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); + if (schema.isDocumentsOnly()) continue; sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } } - private void addCluster(SearchCluster sc) { - if (clusters.containsKey(sc.getClusterName())) { - throw new IllegalArgumentException("Duplicate cluster '" + sc.getClusterName() + "'"); + private void addCluster(IndexedSearchCluster sc) { + if (searchCluster != null) { + throw new IllegalArgumentException("Duplicate indexed cluster '" + searchCluster.getClusterName() + "'"); } - if (sc instanceof IndexedSearchCluster) { - if (indexedCluster != null) { - throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); - } - indexedCluster = (IndexedSearchCluster)sc; - } - clusters.put(sc.getClusterName(), sc); + searchCluster = sc; } /** @@ -280,24 +216,26 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - boolean hasStreaming = false; - boolean hasIndexed = false; - for (var cluster : clusters.values()) { - if (cluster.hasStreaming()) - hasStreaming = true; - else - hasIndexed = true; - } - if (hasIndexed == hasStreaming) return null; + if (searchCluster == null) return false; + boolean hasStreaming = searchCluster.hasStreaming(); + if (searchCluster.hasIndexed() == hasStreaming) return null; return hasStreaming; } + public boolean hasStreaming() { + return (searchCluster != null) && searchCluster.hasStreaming(); + } + + public boolean hasIndexed() { + return (searchCluster != null) && searchCluster.hasIndexed(); + } + public List<SearchNode> getSearchNodes() { - return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; + return (searchCluster != null) ? searchCluster.getSearchNodes() : nonIndexed; } public void addSearchNode(DeployState deployState, ContentNode node, StorageGroup parentGroup, ModelElement element) { - TreeConfigProducer<AnyConfigProducer> parent = hasIndexedCluster() ? getIndexed() : this; + TreeConfigProducer<AnyConfigProducer> parent = (searchCluster != null) ? searchCluster : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; @@ -319,8 +257,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } searchNode.setTls(tls); - if (hasIndexedCluster()) { - getIndexed().addSearcher(searchNode); + if (searchCluster != null) { + searchCluster.addSearcher(searchNode); } else { nonIndexed.add(searchNode); } @@ -347,28 +285,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } public boolean usesHierarchicDistribution() { - return indexedCluster != null && groupToSpecMap.size() > 1; + return searchCluster != null && groupToSpecMap.size() > 1; } public void handleRedundancy(Redundancy redundancy) { this.redundancy = redundancy; } - private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.schemas().get(docType) != null) - .findFirst(); - } - - public List<StreamingSearchCluster> getStreamingClusters() { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .toList(); - } - public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -380,13 +303,15 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - return findStreamingCluster(type.getFullName().getName()).isPresent(); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } private boolean hasIndexingModeIndexed(NewDocumentType type) { - return !hasIndexingModeStreaming(type) - && hasIndexedCluster() - && getIndexed().hasDocumentDB(type.getFullName().getName()); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.INDEX); } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -395,7 +320,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedCluster = false; + boolean hasAnyNonIndexedSchema = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -407,13 +332,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedCluster = true; - findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedSchema = true; + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedCluster = true; + hasAnyNonIndexedSchema = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -422,7 +347,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedCluster) { + if (hasAnyNonIndexedSchema) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); @@ -465,20 +390,19 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(DispatchNodesConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } @Override public void getConfig(DispatchConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } - public Map<String, SearchCluster> getClusters() { return clusters; } - public IndexedSearchCluster getIndexed() { return indexedCluster; } - public boolean hasIndexedCluster() { return indexedCluster != null; } + public IndexedSearchCluster getSearchCluster() { return searchCluster; } + public boolean hasSearchCluster() { return searchCluster != null; } public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public String getClusterName() { return clusterName; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 791faa401ed..bac86e37e8f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -73,14 +73,14 @@ import static java.util.logging.Level.WARNING; * @author bratseth */ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implements - DistributionConfig.Producer, - StorDistributionConfig.Producer, - StorDistributormanagerConfig.Producer, - FleetcontrollerConfig.Producer, - MetricsmanagerConfig.Producer, - MessagetyperouteselectorpolicyConfig.Producer, - BucketspacesConfig.Producer { - + DistributionConfig.Producer, + StorDistributionConfig.Producer, + StorDistributormanagerConfig.Producer, + FleetcontrollerConfig.Producer, + MetricsmanagerConfig.Producer, + MessagetyperouteselectorpolicyConfig.Producer, + BucketspacesConfig.Producer +{ private final String documentSelection; private ContentSearchCluster search; private final boolean isHosted; @@ -140,7 +140,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); - if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) + if (c.search.hasIndexed() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) throw new IllegalArgumentException("Indexed search requires proton as engine"); if (documentsElement != null) { @@ -177,8 +177,9 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (visibilityDelay != null) { csc.setVisibilityDelay(visibilityDelay); } - if (csc.hasIndexedCluster()) { - setupIndexedCluster(csc.getIndexed(), search, element, logger); + IndexedSearchCluster sc = csc.getSearchCluster(); + if (sc != null) { + setupIndexedCluster(sc, search, element, logger); } } @@ -530,7 +531,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem super.validate(); if (search.usesHierarchicDistribution() && !isHosted) { // validate manually configured groups - new IndexedHierarchicDistributionValidator(rootGroup, redundancy, search.getIndexed().getTuning().dispatch.getDispatchPolicy()).validate(); + new IndexedHierarchicDistributionValidator(rootGroup, redundancy, search.getSearchCluster().getTuning().dispatch.getDispatchPolicy()).validate(); } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 7fc713f81ef..f4e6244fa25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -13,7 +13,7 @@ public class EngineFactoryBuilder { public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) { ModelElement persistence = clusterElem.child("engine"); if (persistence != null) { - if (c.getSearch().hasIndexedCluster() && persistence.child("proton") == null) { + if (c.getSearch().hasIndexed() && persistence.child("proton") == null) { throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine."); } |