diff options
Diffstat (limited to 'config-model/src/main/java/com')
6 files changed, 134 insertions, 71 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 180c8e6012f..93bec4975a6 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -709,8 +709,17 @@ public class Schema implements ImmutableSchema { public FieldSets fieldSets() { return fieldSets; } + private Schema inheritedSchema = null; + + public void setInheritedSchema(Schema value) { + inheritedSchema = value; + } + /** Returns the schema inherited by this, or throws if none */ - private Schema requireInherited() { return owner.schemas().get(inherited.get()); } + private Schema requireInherited() { + if (inheritedSchema != null) return inheritedSchema; + return owner.schemas().get(inherited.get()); + } /** * For adding structs defined in document scope diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 0abcc9e890a..40ec84ec8bc 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -98,6 +98,13 @@ public class ConvertParsedSchemas { Schema schema = parsed.getDocumentWithoutSchema() ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + inherited.ifPresent(parentName -> { + for (var possibleParent : resultList) { + if (possibleParent.getName().equals(parentName)) { + schema.setInheritedSchema(possibleParent); + } + } + }); convertSchema(schema, parsed); resultList.add(schema); } @@ -145,7 +152,23 @@ public class ConvertParsedSchemas { docsum.setOmitSummaryFeatures(true); } for (var parsedField : parsed.getSummaryFields()) { - DataType dataType = typeContext.resolveType(parsedField.getType()); + var parsedType = parsedField.getType(); + DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; + var existingField = schema.getField(parsedField.name()); + if (existingField != null) { + var existingType = existingField.getDataType(); + if (dataType == null) { + dataType = existingType; + } else if (!dataType.equals(existingType)) { + if (dataType.getValueClass().equals(com.yahoo.document.datatypes.WeightedSet.class)) { + // "adjusting type for field " + parsedField.name() + " in document-summary " + parsed.name() + " field already has: " + existingType + " but declared type was: " + dataType + dataType = existingType; + } + } + } + if (dataType == null) { + throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name()); + } var summaryField = new SummaryField(parsedField.name(), dataType); // XXX does not belong here: summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java deleted file mode 100644 index d9bf82980d7..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; - -/** - * Validates that a Vespa Cloud application has at least one container cluster. - * - * @author jonmv - */ -public class ContainerInCloudValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - if (deployState.isHosted() && model.getContainerClusters().isEmpty()) - throw new IllegalArgumentException("Vespa Cloud applications must have at least one container cluster"); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 4f2f8e7932c..2576d9cb392 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -76,7 +76,6 @@ public class Validation { new StreamingValidator().validate(model, deployState); new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); - new ContainerInCloudValidator().validate(model, deployState); new DeploymentSpecValidator().validate(model, deployState); new ValidationOverridesValidator().validate(model, deployState); new ConstantValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 5a96e33c522..e2166b263ee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.utils.Duration; import org.w3c.dom.Element; import java.util.Optional; @@ -47,9 +48,29 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc clusterControllerTuning = tuning.child("cluster-controller"); } + var tuningConfig = new ClusterControllerTuningBuilder(clusterControllerTuning, + minNodeRatioPerGroup, + bucketSplittingMinimumBits) + .build(); + if (ancestor instanceof ContentCluster) { + int numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups(); + if (tuningConfig.maxGroupsAllowedDown().isPresent()) { + Integer maxGroupsAllowedDown = tuningConfig.maxGroupsAllowedDown().get(); + if (deployState.zone().environment().isProduction() && (maxGroupsAllowedDown > numberOfLeafGroups)) + throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + maxGroupsAllowedDown + + ") larger than number of groups (" + numberOfLeafGroups + ")"); + } else { + // Reduce to numberOfLeafGroups for tests or in environments where number of groups are reduced by policy (dev, test, staging, perf) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(numberOfLeafGroups); + } + } else { + // Reduce to 1 for tests (ancestor is a mock class) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(1); + } + return new ClusterControllerConfig(ancestor, clusterName, - new ClusterControllerTuning(clusterControllerTuning, minNodeRatioPerGroup, bucketSplittingMinimumBits), + tuningConfig, resourceLimits, allowMoreThanOneContentGroupDown); } @@ -99,45 +120,81 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc resourceLimits.getConfig(builder); } - public ClusterControllerTuning tuning() { return tuning; } - - public static class ClusterControllerTuning { - - private final Optional<Double> minNodeRatioPerGroup; - private final Optional<Duration> initProgressTime; - private final Optional<Duration> transitionTime; - private final Optional<Long> maxPrematureCrashes; - private final Optional<Duration> stableStateTimePeriod; - private final Optional<Double> minDistributorUpRatio; - private final Optional<Double> minStorageUpRatio; - private final Optional<Integer> minSplitBits; - final Optional<Integer> maxGroupsAllowedDown; - - ClusterControllerTuning(ModelElement tuning, - Optional<Double> minNodeRatioPerGroup, - Optional<Integer> bucketSplittingMinimumBits) { - this.minSplitBits = bucketSplittingMinimumBits; - this.minNodeRatioPerGroup = minNodeRatioPerGroup; - if (tuning == null) { - this.initProgressTime = Optional.empty(); - this.transitionTime = Optional.empty(); - this.maxPrematureCrashes = Optional.empty(); - this.stableStateTimePeriod = Optional.empty(); - this.minDistributorUpRatio = Optional.empty(); - this.minStorageUpRatio = Optional.empty(); - this.maxGroupsAllowedDown = Optional.empty(); - } else { - this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); - this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); - this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); - this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); - this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); - this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); - this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); - } + public ClusterControllerTuning tuning() {return tuning;} + +private static class ClusterControllerTuningBuilder { + + private final Optional<Double> minNodeRatioPerGroup; + private final Optional<Duration> initProgressTime; + private final Optional<Duration> transitionTime; + private final Optional<Long> maxPrematureCrashes; + private final Optional<Duration> stableStateTimePeriod; + private final Optional<Double> minDistributorUpRatio; + private final Optional<Double> minStorageUpRatio; + private final Optional<Integer> minSplitBits; + final Optional<Integer> maxGroupsAllowedDown; + + ClusterControllerTuningBuilder(ModelElement tuning, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> bucketSplittingMinimumBits) { + this.minSplitBits = bucketSplittingMinimumBits; + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + if (tuning == null) { + this.initProgressTime = Optional.empty(); + this.transitionTime = Optional.empty(); + this.maxPrematureCrashes = Optional.empty(); + this.stableStateTimePeriod = Optional.empty(); + this.minDistributorUpRatio = Optional.empty(); + this.minStorageUpRatio = Optional.empty(); + this.maxGroupsAllowedDown = Optional.empty(); + } else { + this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); + this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); + this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); + this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); + this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); + this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); + this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); } + } - public Optional<Integer> maxGroupsAllowedDown() { return maxGroupsAllowedDown; } + private ClusterControllerTuning build() { + return new ClusterControllerTuning(initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + maxGroupsAllowedDown, + minNodeRatioPerGroup, + minSplitBits); } } + +private record ClusterControllerTuning(Optional<Duration> initProgressTime, + Optional<Duration> transitionTime, + Optional<Long> maxPrematureCrashes, + Optional<Duration> stableStateTimePeriod, + Optional<Double> minDistributorUpRatio, + Optional<Double> minStorageUpRatio, + Optional<Integer> maxGroupsAllowedDown, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> minSplitBits) { + + public ClusterControllerTuning withMaxGroupsAllowedDown(int maxGroupsAllowedDown) { + return new ClusterControllerConfig.ClusterControllerTuning( + initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + Optional.of(maxGroupsAllowedDown), + minNodeRatioPerGroup, + minSplitBits); + } + +} + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index f1f210b013c..f1d5c7c9220 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -164,18 +164,12 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem DeployState deployState, ContentCluster c, ClusterResourceLimits resourceLimits) { - var config = new ClusterControllerConfig.Builder(c.clusterId, - contentElement, - resourceLimits.getClusterControllerLimits(), - deployState.featureFlags() - .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) + return new ClusterControllerConfig.Builder(c.clusterId, + contentElement, + resourceLimits.getClusterControllerLimits(), + deployState.featureFlags() + .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) .build(deployState, c, contentElement.getXml()); - config.tuning().maxGroupsAllowedDown().ifPresent(m -> { - int numberOfLeafGroups = c.getRootGroup().getNumberOfLeafGroups(); - if (m > numberOfLeafGroups) - throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + m + ") larger than number of groups (" + numberOfLeafGroups + ")"); - }); - return config; } private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) { |