diff options
Diffstat (limited to 'config-model')
8 files changed, 167 insertions, 70 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java index 82ad8e5d6e8..47024c1171c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java @@ -35,7 +35,7 @@ public class RedundancyIncreaseValidator implements ChangeValidator { } private int redundancyOf(ContentCluster cluster) { - return cluster.redundancy().finalRedundancy(); + return cluster.getRedundancy().finalRedundancy(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java index 5228610537f..2be0f0b8422 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.application.validation.first; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; @@ -10,7 +9,6 @@ import com.yahoo.vespa.model.application.validation.Validator; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.time.Instant; import java.util.List; import java.util.stream.Stream; @@ -48,7 +46,7 @@ public class RedundancyValidator extends Validator implements ChangeValidator { } private boolean hasRedundancyOne(ContentCluster cluster) { - return cluster != null && cluster.redundancy().finalRedundancy() == 1 && cluster.redundancy().groups() == 1; + return cluster != null && cluster.getRedundancy().finalRedundancy() == 1 && cluster.getRedundancy().groups() == 1; } private void invalidRedundancy(ContentCluster cluster, DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ec7acaf819f..34ea41384bc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -270,6 +270,25 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> clusters.put(sc.getClusterName(), sc); } + /** + * Returns whether the schemas in this cluster use streaming mode. + * + * @return True if this cluster only has schemas with streaming mode, False if it only has schemas + * with indexing, null if it has both or none. + */ + public Boolean isStreaming() { + boolean hasStreaming = false; + boolean hasIndexed = false; + for (var cluster : clusters.values()) { + if (cluster.isStreaming()) + hasStreaming = true; + else + hasIndexed = true; + } + if (hasIndexed == hasStreaming) return null; + return hasStreaming; + } + public List<SearchNode> getSearchNodes() { return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 52b2ce06dfe..6078215f9b6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -171,12 +171,11 @@ public class StorageGroup { } @Override - public boolean equals(Object obj) { - if (obj instanceof StorageGroup) { - StorageGroup rhs = (StorageGroup)obj; - return this.index.equals(rhs.index) && - this.name.equals(rhs.name) && - this.partitions.equals(rhs.partitions); + public boolean equals(Object o) { + if (o instanceof StorageGroup other) { + return this.index.equals(other.index) && + this.name.equals(other.name) && + this.partitions.equals(other.partitions); } return false; } @@ -208,9 +207,7 @@ public class StorageGroup { this.context = context; } - public StorageGroup buildRootGroup(DeployState deployState, - RedundancyBuilder redundancyBuilder, - ContentCluster owner) { + public StorageGroup buildRootGroup(DeployState deployState, ContentCluster owner, Boolean isStreaming) { try { if (owner.isHosted()) validateRedundancyAndGroups(deployState.zone().environment()); @@ -229,7 +226,8 @@ public class StorageGroup { ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); - Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(), + RedundancyBuilder redundancyBuilder = new RedundancyBuilder(clusterElement); + Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), isStreaming, storageGroup.subgroups.size(), storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(false)); owner.setRedundancy(redundancy); if (storageGroup.partitions.isEmpty() && (redundancy.groups() > 1)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 2592beca6c6..f792ac3a591 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -114,7 +114,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem new SearchDefinitionBuilder().build(deployState.getDocumentModel().getDocumentManager(), documentsElement); String routingSelection = new DocumentSelectionBuilder().build(documentsElement); - RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement); Set<NewDocumentType> globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement); String clusterId = getClusterId(contentElement); @@ -133,7 +132,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); - c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c); + c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming()); c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits); validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); c.search.handleRedundancy(c.redundancy); @@ -447,7 +446,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem public final ContentSearchCluster getSearch() { return search; } - public Redundancy redundancy() { return redundancy; } + public Redundancy getRedundancy() { return redundancy; } public ContentCluster setRedundancy(Redundancy redundancy) { this.redundancy = redundancy; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java index e7bafdf52e4..d310db067a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java @@ -20,7 +20,7 @@ public class RedundancyBuilder { // Always global (across groups) private Integer globalMinRedundancy = null; - RedundancyBuilder(ModelElement clusterXml) { + public RedundancyBuilder(ModelElement clusterXml) { ModelElement redundancyElement = clusterXml.child("redundancy"); if (redundancyElement != null) { initialRedundancy = redundancyElement.integerAttribute("reply-after"); @@ -47,22 +47,40 @@ public class RedundancyBuilder { throw new IllegalArgumentException("Either <redundancy> or <min-redundancy> must be set"); } - public Redundancy build(boolean isHosted, int subGroups, int leafGroups, int totalNodes) { + /** + * @param isHosted + * @param isStreaming true if this cluster only has schemas with streaming mode, false if it only has schemas + * without streaming, null if it has both + * @param subGroups + * @param leafGroups + * @param totalNodes + * @return + */ + public Redundancy build(boolean isHosted, Boolean isStreaming, int subGroups, int leafGroups, int totalNodes) { if (isHosted) { if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy * leafGroups < globalMinRedundancy )) initialRedundancy = finalRedundancy = (int)Math.ceil((double)globalMinRedundancy / leafGroups); if (readyCopies == null) { - if (leafGroups > 1) - readyCopies = 1; - else - readyCopies = finalRedundancy > 1 ? 2 : 1; + if (isStreaming == Boolean.TRUE) { + readyCopies = finalRedundancy; + } + else { // If isStreaming is null (mixed mode cluster) there are no good options ... + if (leafGroups > 1) + readyCopies = 1; + else + readyCopies = finalRedundancy > 1 ? 2 : 1; + } } return new Redundancy(initialRedundancy, finalRedundancy, readyCopies, leafGroups, totalNodes); } else { if (globalMinRedundancy != null && ( finalRedundancy == null || finalRedundancy < globalMinRedundancy)) initialRedundancy = finalRedundancy = globalMinRedundancy; - if (readyCopies == null) - readyCopies = finalRedundancy > 1 ? Math.max(subGroups, 2) : 1; + if (readyCopies == null) { + if (isStreaming == Boolean.TRUE) + readyCopies = finalRedundancy; + else // If isStreaming is null (mixed mode cluster) there are no good options ... + readyCopies = finalRedundancy > 1 ? Math.max(subGroups, 2) : 1; + } subGroups = Math.max(1, subGroups); IndexedHierarchicDistributionValidator.validateThatLeafGroupsCountIsAFactorOfRedundancy(finalRedundancy, subGroups); IndexedHierarchicDistributionValidator.validateThatReadyCopiesIsCompatibleWithRedundancy(finalRedundancy, readyCopies, subGroups); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 84804bc48fa..19fe9e0038d 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1183,9 +1183,9 @@ public class ModelProvisioningTest { ContentCluster cluster = model.getContentClusters().get("bar"); List<StorageGroup> subGroups = cluster.getRootGroup().getSubgroups(); - assertEquals(2*3, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 - assertEquals(2*3, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 - assertEquals(2*3, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 + assertEquals(2*3, cluster.getRedundancy().effectiveInitialRedundancy()); // Reduced from 3*3 + assertEquals(2*3, cluster.getRedundancy().effectiveFinalRedundancy()); // Reduced from 3*4 + assertEquals(2*3, cluster.getRedundancy().effectiveReadyCopies()); // Reduced from 3*3 assertEquals("2|2|*", cluster.getRootGroup().getPartitions().get()); // Reduced from 4|4|* assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(3, subGroups.size()); @@ -1257,9 +1257,9 @@ public class ModelProvisioningTest { assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); - assertEquals(2, cluster.redundancy().effectiveInitialRedundancy()); - assertEquals(2, cluster.redundancy().effectiveFinalRedundancy()); - assertEquals(2, cluster.redundancy().effectiveReadyCopies()); + assertEquals(2, cluster.getRedundancy().effectiveInitialRedundancy()); + assertEquals(2, cluster.getRedundancy().effectiveFinalRedundancy()); + assertEquals(2, cluster.getRedundancy().effectiveReadyCopies()); assertEquals("1|*", cluster.getRootGroup().getPartitions().get()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(2, cluster.getRootGroup().getSubgroups().size()); @@ -1287,9 +1287,9 @@ public class ModelProvisioningTest { ContentCluster cluster = model.getContentClusters().get("bar"); assertEquals(2, cluster.getStorageCluster().getChildren().size()); - assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); - assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); - assertEquals(1, cluster.redundancy().effectiveReadyCopies()); + assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy()); + assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy()); + assertEquals(1, cluster.getRedundancy().effectiveReadyCopies()); assertEquals(2, cluster.getRootGroup().getNodes().size()); assertEquals(0, cluster.getRootGroup().getSubgroups().size()); } @@ -1324,9 +1324,9 @@ public class ModelProvisioningTest { assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); - assertEquals(4, cluster.redundancy().effectiveInitialRedundancy()); - assertEquals(4, cluster.redundancy().effectiveFinalRedundancy()); - assertEquals(4, cluster.redundancy().effectiveReadyCopies()); + assertEquals(4, cluster.getRedundancy().effectiveInitialRedundancy()); + assertEquals(4, cluster.getRedundancy().effectiveFinalRedundancy()); + assertEquals(4, cluster.getRedundancy().effectiveReadyCopies()); assertEquals(4, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size()); assertEquals(4, cluster.getSearch().getIndexed().getSearchableCopies()); assertFalse(cluster.getRootGroup().getPartitions().isPresent()); @@ -1368,9 +1368,9 @@ public class ModelProvisioningTest { assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); - assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 - assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 - assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 + assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy()); // Reduced from 3*3 + assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy()); // Reduced from 3*4 + assertEquals(1, cluster.getRedundancy().effectiveReadyCopies()); // Reduced from 3*3 assertFalse(cluster.getRootGroup().getPartitions().isPresent()); // 1 group - > flattened -> no distribution assertEquals(1, cluster.getRootGroup().getNodes().size()); assertEquals(0, cluster.getRootGroup().getSubgroups().size()); @@ -1473,9 +1473,9 @@ public class ModelProvisioningTest { assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); - assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); - assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); - assertEquals(1, cluster.redundancy().effectiveReadyCopies()); + assertEquals(1, cluster.getRedundancy().effectiveInitialRedundancy()); + assertEquals(1, cluster.getRedundancy().effectiveFinalRedundancy()); + assertEquals(1, cluster.getRedundancy().effectiveReadyCopies()); assertEquals(1, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size()); assertFalse(cluster.getRootGroup().getPartitions().isPresent()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4ce7119f5f7..73bbd6ee464 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -38,6 +38,7 @@ import com.yahoo.vespa.model.routing.DocumentProtocol; import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -470,7 +471,8 @@ public class ContentClusterTest extends ContentBaseTest { new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); fail("Deploying without redundancy should fail"); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Either <redundancy> or <min-redundancy> must be set"), e.getMessage()); + assertEquals("In content cluster 'bar': Either <redundancy> or <min-redundancy> must be set", + Exceptions.toMessageString(e)); } } @@ -478,12 +480,13 @@ public class ContentClusterTest extends ContentBaseTest { void testRedundancyFinalLessThanInitial() { try { parse( - "<content version=\"1.0\" id=\"storage\">\n" + - " <redundancy reply-after=\"4\">2</redundancy>\n" + - " <group>" + - " <node hostalias='node0' distribution-key='0' />" + - " </group>" + - "</content>" + """ + <content version="1.0" id="storage"> + <redundancy reply-after="4">2</redundancy> + <group> + <node hostalias='node0' distribution-key='0' /> + </group> + </content>""" ); fail("no exception thrown"); } catch (Exception e) { /* ignore */ @@ -494,17 +497,18 @@ public class ContentClusterTest extends ContentBaseTest { void testReadyTooHigh() { try { parse( - "<content version=\"1.0\" id=\"storage\">\n" + - " <engine>" + - " <proton>" + - " <searchable-copies>3</searchable-copies>" + - " </proton>" + - " </engine>" + - " <redundancy>2</redundancy>\n" + - " <group>" + - " <node hostalias='node0' distribution-key='0' />" + - " </group>" + - "</content>" + """ + <content version="1.0" id="storage"> + <engine> + <proton> + <searchable-copies>3</searchable-copies> + </proton> + </engine> + <redundancy>2</redundancy> + <group> + <node hostalias='node0' distribution-key='0' /> + </group> + </content>""" ); fail("no exception thrown"); } catch (Exception e) { /* ignore */ @@ -972,15 +976,17 @@ public class ContentClusterTest extends ContentBaseTest { @Test void reserved_document_name_throws_exception() { - String xml = "<content version=\"1.0\" id=\"storage\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"true\" mode=\"index\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>"; + String xml = """ + <content version="1.0" id="storage"> + <redundancy>1</redundancy> + <documents> + <document type="true" mode="index"/> + </documents> + <group> + <node distribution-key="0" hostalias="mockhost"/> + </group> + </content> + """; List<String> sds = ApplicationPackageUtils.generateSchemas("true"); try { @@ -991,6 +997,65 @@ public class ContentClusterTest extends ContentBaseTest { } } + @Test + void default_searchable_copies_indexing() { + String services = """ + <content version="1.0" id="storage"> + <redundancy>3</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <group> + <node distribution-key="0" hostalias="mockhost"/> + <node distribution-key="1" hostalias="mockhost"/> + <node distribution-key="2" hostalias="mockhost"/> + </group> + </content> + """; + var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("music")).create(); + assertEquals(2, model.getContentClusters().get("storage").getRedundancy().readyCopies()); + } + + @Test + void default_searchable_copies_streaming() { + String services = """ + <content version="1.0" id="storage"> + <redundancy>3</redundancy> + <documents> + <document type="mail" mode="streaming"/> + </documents> + <group> + <node distribution-key="0" hostalias="mockhost"/> + <node distribution-key="1" hostalias="mockhost"/> + <node distribution-key="2" hostalias="mockhost"/> + </group> + </content> + """; + var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("mail")).create(); + assertEquals(3, model.getContentClusters().get("storage").getRedundancy().readyCopies()); + } + + /** Here there is no good choice. */ + @Test + void default_searchable_copies_mixed() { + String services = """ + <content version="1.0" id="storage"> + <redundancy>3</redundancy> + <documents> + <document type="music" mode="index"/> + <document type="mail" mode="streaming"/> + </documents> + <group> + <node distribution-key="0" hostalias="mockhost"/> + <node distribution-key="1" hostalias="mockhost"/> + <node distribution-key="2" hostalias="mockhost"/> + </group> + </content> + """; + var model = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSchemas("music", "mail")).create(); + assertEquals(2, model.getContentClusters().get("storage").getRedundancy().readyCopies()); + } + private void assertClusterHasBucketSpaceMappings(AllClustersBucketSpacesConfig config, String clusterId, List<String> defaultSpaceTypes, List<String> globalSpaceTypes) { AllClustersBucketSpacesConfig.Cluster cluster = config.cluster(clusterId); |