diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java')
| -rw-r--r-- | config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java | 188 |
1 files changed, 56 insertions, 132 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a92362bf899..ebb556a2ba2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -31,9 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.Objects; -import java.util.TreeMap; import java.util.function.Predicate; /** @@ -53,11 +50,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> private final boolean flushOnShutdown; private final Boolean syncTransactionLog; - /** If this is set up for streaming search, it is modelled as one search cluster per search definition */ - private final Map<String, SearchCluster> clusters = new TreeMap<>(); - /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ - private IndexedSearchCluster indexedCluster; + private IndexedSearchCluster searchCluster; private final IndexingDocproc indexingDocproc; private Redundancy redundancy; @@ -115,8 +109,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); - buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); + buildSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -128,73 +121,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { - var schemas = deployState.getSchemas(); - for (var candidate : schemas) { - if (candidate.getName().equals(docTypeName)) { - return candidate; - } - } - return null; - } - - private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) return; - if (docElem == null) { - return; - } - - for (ModelElement docType : docElem.subElements("document")) { - String docTypeName = docType.stringAttribute("type"); - String mode = docType.stringAttribute("mode"); - var schema = findResponsibleSchema(deployState, docTypeName); - if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { - buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); - } - } - } - - private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, - ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.stringAttribute("type"); - StreamingSearchCluster cluster = new StreamingSearchCluster(search, - clusterName + "." + docTypeName, - 0, - docTypeName, - clusterName); - search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); - } - - private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); - if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); - } - - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); - } - } - - private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { - List<ModelElement> indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) { - return indexedDefs; + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); } - for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.stringAttribute("mode"); - if ("index".equals(mode)) { - indexedDefs.add(docType); - } - } - return indexedDefs; + var isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); } } @@ -233,21 +171,24 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public void setVisibilityDelay(double delay) { this.visibilityDelay=delay; - if (hasIndexedCluster()) { - indexedCluster.setVisibilityDelay(delay); + if (searchCluster != null) { + searchCluster.setVisibilityDelay(delay); } } - private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { + private void addSearchCluster(DeployState deployState, IndexedSearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { addSchemas(deployState, documentDefs, cluster); if (queryTimeout != null) { cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - addCluster(cluster); + if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { + addCluster(cluster); + } } + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -255,22 +196,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); + if (schema.isDocumentsOnly()) continue; sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } } - private void addCluster(SearchCluster sc) { - if (clusters.containsKey(sc.getClusterName())) { - throw new IllegalArgumentException("Duplicate cluster '" + sc.getClusterName() + "'"); + private void addCluster(IndexedSearchCluster sc) { + if (searchCluster != null) { + throw new IllegalArgumentException("Duplicate indexed cluster '" + searchCluster.getClusterName() + "'"); } - if (sc instanceof IndexedSearchCluster) { - if (indexedCluster != null) { - throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); - } - indexedCluster = (IndexedSearchCluster)sc; - } - clusters.put(sc.getClusterName(), sc); + searchCluster = sc; } /** @@ -280,24 +216,26 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - boolean hasStreaming = false; - boolean hasIndexed = false; - for (var cluster : clusters.values()) { - if (cluster.hasStreaming()) - hasStreaming = true; - else - hasIndexed = true; - } - if (hasIndexed == hasStreaming) return null; + if (searchCluster == null) return false; + boolean hasStreaming = searchCluster.hasStreaming(); + if (searchCluster.hasIndexed() == hasStreaming) return null; return hasStreaming; } + public boolean hasStreaming() { + return (searchCluster != null) && searchCluster.hasStreaming(); + } + + public boolean hasIndexed() { + return (searchCluster != null) && searchCluster.hasIndexed(); + } + public List<SearchNode> getSearchNodes() { - return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; + return (searchCluster != null) ? searchCluster.getSearchNodes() : nonIndexed; } public void addSearchNode(DeployState deployState, ContentNode node, StorageGroup parentGroup, ModelElement element) { - TreeConfigProducer<AnyConfigProducer> parent = hasIndexedCluster() ? getIndexed() : this; + TreeConfigProducer<AnyConfigProducer> parent = (searchCluster != null) ? searchCluster : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; @@ -319,8 +257,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } searchNode.setTls(tls); - if (hasIndexedCluster()) { - getIndexed().addSearcher(searchNode); + if (searchCluster != null) { + searchCluster.addSearcher(searchNode); } else { nonIndexed.add(searchNode); } @@ -347,28 +285,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } public boolean usesHierarchicDistribution() { - return indexedCluster != null && groupToSpecMap.size() > 1; + return searchCluster != null && groupToSpecMap.size() > 1; } public void handleRedundancy(Redundancy redundancy) { this.redundancy = redundancy; } - private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.schemas().get(docType) != null) - .findFirst(); - } - - public List<StreamingSearchCluster> getStreamingClusters() { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .toList(); - } - public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -380,13 +303,15 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - return findStreamingCluster(type.getFullName().getName()).isPresent(); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } private boolean hasIndexingModeIndexed(NewDocumentType type) { - return !hasIndexingModeStreaming(type) - && hasIndexedCluster() - && getIndexed().hasDocumentDB(type.getFullName().getName()); + if (searchCluster == null) return false; + var schemaInfo = searchCluster.schemas().get(type.getName()); + return (schemaInfo != null) && (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.INDEX); } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -395,7 +320,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedCluster = false; + boolean hasAnyNonIndexedSchema = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -407,13 +332,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedCluster = true; - findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedSchema = true; + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + searchCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedCluster = true; + hasAnyNonIndexedSchema = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -422,7 +347,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedCluster) { + if (hasAnyNonIndexedSchema) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); @@ -465,20 +390,19 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(DispatchNodesConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } @Override public void getConfig(DispatchConfig.Builder builder) { - if (hasIndexedCluster()) { - getIndexed().getConfig(builder); + if (searchCluster != null) { + searchCluster.getConfig(builder); } } - public Map<String, SearchCluster> getClusters() { return clusters; } - public IndexedSearchCluster getIndexed() { return indexedCluster; } - public boolean hasIndexedCluster() { return indexedCluster != null; } + public IndexedSearchCluster getSearchCluster() { return searchCluster; } + public boolean hasSearchCluster() { return searchCluster != null; } public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public String getClusterName() { return clusterName; } |