diff options
Diffstat (limited to 'config-model/src/main')
19 files changed, 117 insertions, 298 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 2499f2f625d..25d45dee234 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.codegen.CNode; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; @@ -19,7 +18,6 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; @@ -27,8 +25,7 @@ import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; -import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; @@ -46,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.logging.Logger; /** * Contains various state during deploy that should be available in all builders of a {@link com.yahoo.config.model.ConfigModel} @@ -71,6 +67,7 @@ public class DeployState implements ConfigDefinitionStore { private final SemanticRules semanticRules; private final ValidationOverrides validationOverrides; private final Version wantedNodeVespaVersion; + private final Instant now; private final HostProvisioner provisioner; @@ -102,8 +99,9 @@ public class DeployState implements ConfigDefinitionStore { this.zone = zone; this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated - this.validationOverrides = new ValidationOverridesXMLReader().read(applicationPackage.getValidationOverrides(), now); + this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; + this.now = now; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -213,6 +211,8 @@ public class DeployState implements ConfigDefinitionStore { public SemanticRules getSemanticRules() { return semanticRules; } public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; } + + public Instant now() { return now; } public static class Builder { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 1be22be1947..c61435ca831 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -29,8 +29,8 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.config.GenericConfig; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.builder.VespaModelBuilder; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.clients.Clients; @@ -51,6 +51,7 @@ import java.io.File; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Constructor; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -212,14 +213,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public ApplicationConfigProducerRoot getVespa() { return root; } @Override - public boolean allowModelVersionMismatch() { - return validationOverrides.allows(ValidationId.configModelVersionMismatch) || - validationOverrides.allows(ValidationId.skipOldConfigModels); // implies this + public boolean allowModelVersionMismatch(Instant now) { + return validationOverrides.allows(ValidationId.configModelVersionMismatch, now) || + validationOverrides.allows(ValidationId.skipOldConfigModels, now); // implies this } @Override - public boolean skipOldConfigModels() { - return validationOverrides.allows(ValidationId.skipOldConfigModels); + public boolean skipOldConfigModels(Instant now) { + return validationOverrides.allows(ValidationId.skipOldConfigModels, now); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d1b8ae63641..490627e7b45 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; @@ -15,6 +16,7 @@ import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterC import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -56,13 +58,14 @@ public class Validation { Optional<Model> currentActiveModel = deployState.getPreviousModel(); if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) return validateChanges((VespaModel)currentActiveModel.get(), model, - deployState.validationOverrides(), deployState.getDeployLogger()); + deployState.validationOverrides(), deployState.getDeployLogger(), deployState.now()); else return new ArrayList<>(); } private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel, - ValidationOverrides overrides, DeployLogger logger) { + ValidationOverrides overrides, DeployLogger logger, + Instant now) { ChangeValidator[] validators = new ChangeValidator[] { new IndexingModeChangeValidator(), new IndexedSearchClusterChangeValidator(), @@ -73,7 +76,7 @@ public class Validation { new ContainerRestartValidator(), }; return Arrays.stream(validators) - .flatMap(v -> v.validate(currentModel, nextModel, overrides).stream()) + .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) .collect(toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java deleted file mode 100644 index 8f9b0081db1..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import java.util.Optional; - -/** - * Ids of validations that can be overridden - * - * @author bratseth - */ -public enum ValidationId { - - indexingChange("indexing-change"), // Changing what tokens are expected and stored in field indexes - indexModeChange("indexing-mode-change"), // Changing the index mode (streaming, indexed, store-only) of documents - fieldTypeChange("field-type-change"), // Field type changes - clusterSizeReduction("cluster-size-reduction"), // Large reductions in cluster size - contentClusterRemoval("content-cluster-removal"), // Removal (or id change) of content clusters - deploymentRemoval("deployment-removal"), // Removal of production zones from deployment.xml - skipAutomaticTenantUpgradeTests("skip-automatic-tenant-upgrade-test"), // Skip platform supplied staging tests - configModelVersionMismatch("config-model-version-mismatch"), // Internal use - skipOldConfigModels("skip-old-config-models"), // Internal use - forceAutomaticTenantUpgradeTests("force-automatic-tenant-upgrade-test"); // Internal use - - private final String id; - - ValidationId(String id) { this.id = id; } - - public String value() { return id; } - - @Override - public String toString() { return id; } - - /** - * Returns the validation id from this string. - * Use this instead of valueOf to match string on the (canonical) dash-separated form. - * - * @return the matching validation id or empty if none - */ - public static Optional<ValidationId> from(String id) { - for (ValidationId candidate : ValidationId.values()) - if (id.equals(candidate.toString())) return Optional.of(candidate); - return Optional.empty(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java deleted file mode 100644 index 25464ce40ba..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.google.common.collect.ImmutableList; - -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -/** - * A set of allows which suppresses specific validations in limited time periods. - * This is useful to be able to complete a deployment in cases where the application - * owner believes that the changes to be deployed have acceptable consequences. - * Immutable. - * - * @author bratseth - */ -public class ValidationOverrides { - - private final List<Allow> overrides; - - /** Instant to use as "now". This is a field to allow unit testing. */ - private final Instant now; - - /** Creates validation overrides for the current instant */ - public ValidationOverrides(List<Allow> overrides) { - this(overrides, Instant.now()); - } - - public ValidationOverrides(List<Allow> overrides, Instant now) { - this.overrides = ImmutableList.copyOf(overrides); - this.now = now; - for (Allow override : overrides) - if (now.plus(Duration.ofDays(30)).isBefore(override.until)) - throw new IllegalArgumentException(override + " is too far in the future: Max 30 days is allowed"); - } - - /** Throws a ValidationException unless this validation is overridden at this time */ - public void invalid(ValidationId validationId, String message) { - if ( ! allows(validationId)) - throw new ValidationException(validationId, message); - } - - public boolean allows(String validationIdString) { - Optional<ValidationId> validationId = ValidationId.from(validationIdString); - if ( ! validationId.isPresent()) return false; // unknown id -> not allowed - return allows(validationId.get()); - } - - /** Returns whether the given (assumed invalid) change is allowed by this at the moment */ - public boolean allows(ValidationId validationId) { - for (Allow override : overrides) - if (override.allows(validationId, now)) - return true; - return false; - } - - public static ValidationOverrides empty() { return new ValidationOverrides(ImmutableList.of()); } - - /** A validation override which allows a particular change. Immutable. */ - public static class Allow { - - private final ValidationId validationId; - private final Instant until; - - public Allow(ValidationId validationId, Instant until) { - this.validationId = validationId; - this.until = until; - } - - public boolean allows(ValidationId validationId, Instant now) { - return this.validationId.equals(validationId) && now.isBefore(until); - } - - @Override - public String toString() { return "allow '" + validationId + "' until " + until; } - - } - - /** - * A deployment validation exception. - * Deployment validations can be {@link ValidationOverrides overridden} based on their id. - * The purpose of this exception is to model that id as a separate field. - */ - public static class ValidationException extends IllegalArgumentException { - - private final ValidationId validationId; - - private ValidationException(ValidationId validationId, String message) { - super(message); - this.validationId = validationId; - } - - /** Returns the unique id of this validation, which can be used to {@link ValidationOverrides override} it */ - public ValidationId validationId() { return validationId; } - - /** Returns "validationId: message" */ - @Override - public String getMessage() { return validationId + ": " + super.getMessage(); } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java index 72eda0ae3ec..bec7fd1518f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java @@ -3,8 +3,9 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; +import java.time.Instant; import java.util.List; /** @@ -22,9 +23,10 @@ public interface ChangeValidator { * @param current the current active model * @param next the next model we would like to activate * @param overrides validation overrides + * @param now the instant to use as now * @return a list of actions specifying what needs to be done in order to activate the new model. * Return an empty list if nothing needs to be done */ - List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides); + List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java index 056b1a80d23..d14fe91a53b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java @@ -3,11 +3,12 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import java.time.Instant; import java.util.Collections; import java.util.List; @@ -19,14 +20,15 @@ import java.util.List; public class ClusterSizeReductionValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { for (ContainerCluster currentCluster : current.getContainerClusters().values()) { ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName()); if (nextCluster == null) continue; validate(currentCluster.getContainers().size(), nextCluster.getContainers().size(), currentCluster.getName(), - overrides); + overrides, + now); } for (ContentCluster currentCluster : current.getContentClusters().values()) { @@ -35,18 +37,20 @@ public class ClusterSizeReductionValidator implements ChangeValidator { validate(currentCluster.getSearch().getSearchNodes().size(), nextCluster.getSearch().getSearchNodes().size(), currentCluster.getName(), - overrides); + overrides, + now); } return Collections.emptyList(); } - private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides) { + private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides, Instant now) { // don't allow more than 50% reduction, but always allow to reduce size with 1 if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1) overrides.invalid(ValidationId.clusterSizeReduction, "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize + - ", new size: " + nextSize + ". New size must be at least 50% of the current size"); + ", new size: " + nextSize + ". New size must be at least 50% of the current size", + now); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index d1d5a1cbfcb..3a97bf4b876 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -9,12 +9,11 @@ import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.RestartConfigs; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; -import com.yahoo.vespa.model.application.validation.change.ChangeValidator; -import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import org.apache.commons.lang3.ClassUtils; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -42,7 +41,7 @@ public class ConfigValueChangeValidator implements ChangeValidator { /** Inspects the configuration in the new and old Vespa model to determine which services that require restart */ @Override public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, - ValidationOverrides overrides) { + ValidationOverrides overrides, Instant now) { return findConfigChangesFromModels(currentModel, nextModel).collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java index c2286264771..c27600578b5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -4,9 +4,10 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.container.Container; +import java.time.Instant; import java.util.List; import static java.util.stream.Collectors.toList; @@ -19,7 +20,8 @@ import static java.util.stream.Collectors.toList; public class ContainerRestartValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides ignored) { + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides ignored, + Instant now) { return nextModel.getContainerClusters().values().stream() .flatMap(cluster -> cluster.getContainers().stream()) .filter(container -> isExistingContainer(container, currentModel)) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java index 5441592b391..52253411419 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -3,10 +3,11 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import java.time.Instant; import java.util.Collections; import java.util.List; @@ -19,13 +20,14 @@ import java.util.List; public class ContentClusterRemovalValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { for (String currentClusterId : current.getContentClusters().keySet()) { ContentCluster nextCluster = next.getContentClusters().get(currentClusterId); if (nextCluster == null) overrides.invalid(ValidationId.contentClusterRemoval, "Content cluster '" + currentClusterId + "' is removed. " + - "This will cause loss of all data in this cluster"); + "This will cause loss of all data in this cluster", + now); } return Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index ca8bc7a3eeb..4ed28331467 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -5,13 +5,14 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.search.DocumentDatabaseChangeValidator; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import java.time.Instant; import java.util.*; import java.util.stream.Collectors; @@ -24,12 +25,12 @@ import java.util.stream.Collectors; public class IndexedSearchClusterChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { List<ConfigChangeAction> result = new ArrayList<>(); for (Map.Entry<String, ContentCluster> currentEntry : current.getContentClusters().entrySet()) { ContentCluster nextCluster = next.getContentClusters().get(currentEntry.getKey()); if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) { - result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides)); + result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides, now)); } } return result; @@ -37,15 +38,17 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster, - ValidationOverrides overrides) { + ValidationOverrides overrides, + Instant now) { List<ConfigChangeAction> result = new ArrayList<>(); - result.addAll(validateDocumentDatabases(currentCluster, nextCluster, overrides)); + result.addAll(validateDocumentDatabases(currentCluster, nextCluster, overrides, now)); return result; } private static List<ConfigChangeAction> validateDocumentDatabases(ContentCluster currentCluster, ContentCluster nextCluster, - ValidationOverrides overrides) { + ValidationOverrides overrides, + Instant now) { List<ConfigChangeAction> result = new ArrayList<>(); for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch())) { String docTypeName = currentDb.getName(); @@ -53,7 +56,7 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { filter(db -> db.getName().equals(docTypeName)).findFirst(); if (nextDb.isPresent()) { result.addAll(validateDocumentDatabase(currentCluster, nextCluster, docTypeName, - currentDb, nextDb.get(), overrides)); + currentDb, nextDb.get(), overrides, now)); } } return result; @@ -64,11 +67,12 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { String docTypeName, DocumentDatabase currentDb, DocumentDatabase nextDb, - ValidationOverrides overrides) { + ValidationOverrides overrides, + Instant now) { NewDocumentType currentDocType = currentCluster.getDocumentDefinitions().get(docTypeName); NewDocumentType nextDocType = nextCluster.getDocumentDefinitions().get(docTypeName); List<VespaConfigChangeAction> result = - new DocumentDatabaseChangeValidator(currentDb, currentDocType, nextDb, nextDocType).validate(overrides); + new DocumentDatabaseChangeValidator(currentDb, currentDocType, nextDb, nextDocType).validate(overrides, now); return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index e876630a401..908809b4b44 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -3,12 +3,11 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; -import com.yahoo.vespa.model.application.validation.change.ChangeValidator; -import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -22,13 +21,14 @@ import java.util.Optional; public class IndexingModeChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides overrides) { + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, + ValidationOverrides overrides, Instant now) { List<ConfigChangeAction> actions = new ArrayList<>(); for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) { ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey()); if (nextCluster == null) continue; - Optional<ConfigChangeAction> change = validateContentCluster(currentEntry.getValue(), nextCluster, overrides); + Optional<ConfigChangeAction> change = validateContentCluster(currentEntry.getValue(), nextCluster, overrides, now); if (change.isPresent()) actions.add(change.get()); } @@ -36,16 +36,17 @@ public class IndexingModeChangeValidator implements ChangeValidator { } private Optional<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster, - ValidationOverrides overrides) { - final boolean currentClusterIsIndexed = currentCluster.getSearch().hasIndexedCluster(); - final boolean nextClusterIsIndexed = nextCluster.getSearch().hasIndexedCluster(); + ValidationOverrides overrides, Instant now) { + boolean currentClusterIsIndexed = currentCluster.getSearch().hasIndexedCluster(); + boolean nextClusterIsIndexed = nextCluster.getSearch().hasIndexedCluster(); if (currentClusterIsIndexed == nextClusterIsIndexed) return Optional.empty(); return Optional.of(VespaRefeedAction.of(ValidationId.indexModeChange.value(), overrides, "Cluster '" + currentCluster.getName() + "' changed indexing mode from '" + - indexingMode(currentClusterIsIndexed) + "' to '" + indexingMode(nextClusterIsIndexed) + "'")); + indexingMode(currentClusterIsIndexed) + "' to '" + indexingMode(nextClusterIsIndexed) + "'", + now)); } private String indexingMode(boolean isIndexed) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java index 7ead547e042..840ab69ba08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java @@ -5,10 +5,9 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; -import com.yahoo.vespa.model.application.validation.change.ChangeValidator; -import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; +import com.yahoo.config.application.api.ValidationOverrides; +import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -25,7 +24,7 @@ public class StartupCommandChangeValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, - ValidationOverrides overrides) { + ValidationOverrides overrides, Instant now) { return findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index 502b1b42aba..0fd38e5dbdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -3,8 +3,9 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; +import java.time.Instant; import java.util.Collections; import java.util.List; @@ -26,29 +27,31 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config private final String documentType; private final boolean allowed; + private final Instant now; - private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed, Instant now) { super(message, services); this.name = name; this.documentType = documentType; this.allowed = allowed; + this.now = now; } /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly - public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message) { - return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name)); + public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, Instant now) { + return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name, now), now); } /** Creates a refeed action */ public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType) { - return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name)); + List<ServiceInfo> services, String documentType, Instant now) { + return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now), now); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed); + return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed, now); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index e534bdfc225..95ae03d4e5f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -5,11 +5,12 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.IndexSchema; import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -46,12 +47,12 @@ public class AttributeChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(final ValidationOverrides overrides) { + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateAddAttributeAspect()); result.addAll(validateRemoveAttributeAspect()); result.addAll(validateAttributeSettings()); - result.addAll(validateTensorTypes(overrides)); + result.addAll(validateTensorTypes(overrides, now)); return result; } @@ -94,7 +95,7 @@ public class AttributeChangeValidator { return result; } - private List<VespaConfigChangeAction> validateTensorTypes(final ValidationOverrides overrides) { + private List<VespaConfigChangeAction> validateTensorTypes(final ValidationOverrides overrides, Instant now) { final List<VespaConfigChangeAction> result = new ArrayList<>(); for (final Attribute nextAttr : nextFields.attributes()) { @@ -109,7 +110,7 @@ public class AttributeChangeValidator { // Tensor attribute has changed type if (!nextAttr.tensorType().get().equals(currentAttr.tensorType().get())) { - result.add(createTensorTypeChangedRefeedAction(currentAttr, nextAttr, overrides)); + result.add(createTensorTypeChangedRefeedAction(currentAttr, nextAttr, overrides, now)); } } } @@ -117,7 +118,7 @@ public class AttributeChangeValidator { return result; } - private static VespaRefeedAction createTensorTypeChangedRefeedAction(Attribute currentAttr, Attribute nextAttr, ValidationOverrides overrides) { + private static VespaRefeedAction createTensorTypeChangedRefeedAction(Attribute currentAttr, Attribute nextAttr, ValidationOverrides overrides, Instant now) { return VespaRefeedAction.of( "tensor-type-change", overrides, @@ -125,7 +126,7 @@ public class AttributeChangeValidator { .addChange( "tensor type", currentAttr.tensorType().get().toString(), - nextAttr.tensorType().get().toString()).build()); + nextAttr.tensorType().get().toString()).build(), now); } private static void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index dd5aa6cf363..bd287f83a1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.search.DocumentDatabase; +import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -32,29 +33,29 @@ public class DocumentDatabaseChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(final ValidationOverrides overrides) { + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); - result.addAll(validateAttributeChanges(overrides)); - result.addAll(validateIndexingScriptChanges(overrides)); - result.addAll(validateDocumentTypeChanges(overrides)); + result.addAll(validateAttributeChanges(overrides, now)); + result.addAll(validateIndexingScriptChanges(overrides, now)); + result.addAll(validateDocumentTypeChanges(overrides, now)); return result; } - private List<VespaConfigChangeAction> validateAttributeChanges(final ValidationOverrides overrides) { + private List<VespaConfigChangeAction> validateAttributeChanges(ValidationOverrides overrides, Instant now) { return new AttributeChangeValidator( currentDatabase.getDerivedConfiguration().getAttributeFields(), currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType, nextDatabase.getDerivedConfiguration().getAttributeFields(), - nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate(overrides); + nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate(overrides, now); } - private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides) { + private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides, Instant now) { return new IndexingScriptChangeValidator(currentDatabase.getDerivedConfiguration().getSearch(), - nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides); + nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides, now); } - private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides) { - return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides); + private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides, Instant now) { + return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides, now); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java index 745ecd58d34..4fd5e82e93f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -4,10 +4,11 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.document.Field; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -132,7 +133,7 @@ public class DocumentTypeChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) { + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { return currentDocType.getAllFields().stream(). map(field -> createFieldChange(field, nextDocType)). filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()). @@ -140,7 +141,8 @@ public class DocumentTypeChangeValidator { overrides, new ChangeMessageBuilder(fieldChange.fieldName()). addChange("data type", fieldChange.currentTypeName(), - fieldChange.nextTypeName()).build())). + fieldChange.nextTypeName()).build(), + now)). collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index cb3f0d40742..ff9230b34f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -7,11 +7,12 @@ import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -32,27 +33,27 @@ public class IndexingScriptChangeValidator { this.nextSearch = nextSearch; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) { + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); for (SDField nextField : nextSearch.allConcreteFields()) { String fieldName = nextField.getName(); SDField currentField = currentSearch.getConcreteField(fieldName); if (currentField != null) { - validateScripts(currentField, nextField, overrides).ifPresent(r -> result.add(r)); + validateScripts(currentField, nextField, overrides, now).ifPresent(r -> result.add(r)); } } return result; } private Optional<VespaConfigChangeAction> validateScripts(SDField currentField, SDField nextField, - ValidationOverrides overrides) { + ValidationOverrides overrides, Instant now) { ScriptExpression currentScript = currentField.getIndexingScript(); ScriptExpression nextScript = nextField.getIndexingScript(); if ( ! equalScripts(currentScript, nextScript)) { ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName()); new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder); messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString()); - return Optional.of(VespaRefeedAction.of(ValidationId.indexingChange.value(), overrides, messageBuilder.build())); + return Optional.of(VespaRefeedAction.of(ValidationId.indexingChange.value(), overrides, messageBuilder.build(), now)); } return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java deleted file mode 100644 index 75a7dc84eaa..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.xml; - -import com.yahoo.text.XML; -import com.yahoo.vespa.model.application.validation.ValidationId; -import com.yahoo.vespa.model.application.validation.ValidationOverrides; -import org.w3c.dom.Element; - -import java.io.Reader; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * Reader of the validation-allows.xml file in application packages. - * - * @author bratseth - */ -public class ValidationOverridesXMLReader { - - /** - * Returns a ValidationOverrides instance with the content of the given Reader. - * An empty ValidationOverrides is returned if the argument is empty. - * - * @param reader the reader which optionally contains a validation-overrides XML structure - * @param now the instant to use as "now", settable for unit testing - * @return a ValidationOverrides from the argument - * @throws IllegalArgumentException if the validation-allows.xml file exists but is invalid - */ - public ValidationOverrides read(Optional<Reader> reader, Instant now) { - if ( ! reader.isPresent()) return ValidationOverrides.empty(); - - try { - // Assume valid structure is ensured by schema validation - Element root = XML.getDocument(reader.get()).getDocumentElement(); - List<ValidationOverrides.Allow> overrides = new ArrayList<>(); - for (Element allow : XML.getChildren(root, "allow")) { - Instant until = LocalDate.parse(allow.getAttribute("until"), DateTimeFormatter.ISO_DATE) - .atStartOfDay().atZone(ZoneOffset.UTC).toInstant() - .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date - Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow)); - if (validationId.isPresent()) // skip unknonw ids as they may be valid for other model versions - overrides.add(new ValidationOverrides.Allow(validationId.get(), until)); - } - return new ValidationOverrides(overrides, now); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("validation-overrides is invalid", e); - } - } - -} |