diff options
Diffstat (limited to 'config-model')
103 files changed, 1218 insertions, 808 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 1c3c088e1fc..7f874d04f13 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -55,7 +55,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false); } /** @@ -65,7 +65,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { * argument is live. Which means that this object will be inconsistent if the given * schema is later modified. */ - public DerivedConfiguration(Schema schema, DeployState deployState) { + public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -81,7 +81,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); indexingScript = new IndexingScript(schema); - indexInfo = new IndexInfo(schema); + indexInfo = new IndexInfo(schema, isStreaming); schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 96c64ff1a1c..7532dec5187 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -55,12 +55,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_INTEGER = "integer"; private static final String CMD_STRING = "string"; private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; + private final boolean isStreaming; private final Set<IndexCommand> commands = new java.util.LinkedHashSet<>(); private final Map<String, String> aliases = new java.util.LinkedHashMap<>(); private final Map<String, FieldSet> fieldSets; private Schema schema; - public IndexInfo(Schema schema) { + public IndexInfo(Schema schema, boolean isStreaming) { + this.isStreaming = isStreaming; this.fieldSets = schema.fieldSets().userFieldSets(); addIndexCommand("sddocname", CMD_INDEX); addIndexCommand("sddocname", CMD_WORD); @@ -223,7 +225,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } private boolean normalizeAccents(ImmutableSDField field) { - return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); + return !isStreaming && field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); } private boolean isTypeOrNested(ImmutableSDField field, DataType type) { @@ -370,7 +372,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { anyStemming = true; stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); } - if (field.getNormalizing().doRemoveAccents()) { + if (normalizeAccents(field)) { anyNormalizing = true; } if (isTypeOrNested(field, DataType.STRING)) { diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 922643b03df..0b542f134ad 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -31,6 +31,8 @@ public class Matching implements Cloneable, Serializable { /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ private Integer maxLength; + /** Maximum number of occurrences for each term */ + private Integer maxTermOccurrences; private String exactMatchTerminator = null; @@ -53,6 +55,8 @@ public class Matching implements Cloneable, Serializable { public Integer maxLength() { return maxLength; } public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } + public Integer maxTermOccurrences() { return maxTermOccurrences; } + public Matching maxTermOccurrences(int maxTermOccurrences) { this.maxTermOccurrences = maxTermOccurrences; return this; } public boolean isTypeUserSet() { return typeUserSet; } public MatchAlgorithm getAlgorithm() { return algorithm; } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index b5d091cca8d..e3ca0090408 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -42,6 +42,7 @@ public class ConvertParsedFields { parsed.getMatchCase().ifPresent(casing -> field.setMatchingCase(casing)); parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); + parsed.getMaxTermOccurrences().ifPresent(maxTermOccurrences -> field.getMatching().maxTermOccurrences(maxTermOccurrences)); parsed.getMatchAlgorithm().ifPresent (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); parsed.getExactTerminator().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index 4c2119b1eba..c7d1a215ce3 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -22,6 +22,7 @@ public class ParsedMatchSettings { private String exactTerminator = null; private Integer gramSize = null; private Integer maxLength = null; + private Integer maxTermOccurrences = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } @@ -29,6 +30,7 @@ public class ParsedMatchSettings { Optional<String> getExactTerminator() { return Optional.ofNullable(exactTerminator); } Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } Optional<Integer> getMaxLength() { return Optional.ofNullable(maxLength); } + Optional<Integer> getMaxTermOccurrences() { return Optional.ofNullable(maxTermOccurrences); } // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } @@ -37,5 +39,6 @@ public class ParsedMatchSettings { void setExactTerminator(String value) { this.exactTerminator = value; } void setGramSize(int value) { this.gramSize = value; } void setMaxLength(int value) { this.maxLength = value; } + void setMaxTermOccurrences(int value) { this.maxTermOccurrences = value; } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index 8ae3ec7a3fa..e6fed35b821 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -16,14 +16,10 @@ import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.Set; -import java.util.TreeSet; - /** * @author Simon Thoresen Hult */ @@ -64,8 +60,16 @@ public class TextMatch extends Processor { } ret.setStemMode(activeStemming.toStemMode()); ret.setRemoveAccents(field.getNormalizing().doRemoveAccents()); - if ((field.getMatching() != null) && (field.getMatching().maxLength() != null)) { - ret.setMaxTokenLength(field.getMatching().maxLength()); + var fieldMatching = field.getMatching(); + if (fieldMatching != null) { + var maxLength = fieldMatching.maxLength(); + if (maxLength != null) { + ret.setMaxTokenLength(maxLength); + } + var maxTermOccurrences = fieldMatching.maxTermOccurrences(); + if (maxTermOccurrences != null) { + ret.setMaxTermOccurrences(maxTermOccurrences); + } } return ret; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java index cc574db2454..12c482f3fdb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -7,7 +7,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.text.XML; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.StringReader; import java.nio.file.Paths; import java.util.Optional; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.jar.JarFile; import java.util.jar.Manifest; @@ -31,36 +32,47 @@ import java.util.regex.Pattern; * * @author bjorncs */ -public abstract class AbstractBundleValidator extends Validator { +public abstract class AbstractBundleValidator implements Validator { - protected abstract void validateManifest(DeployState state, JarFile jar, Manifest mf); - protected abstract void validatePomXml(DeployState state, JarFile jar, Document pom); + protected interface JarContext { + void illegal(String error); + void illegal(String error, Throwable cause); + DeployState deployState(); + static JarContext of(Context context) { + return new JarContext() { + @Override public void illegal(String error) { context.illegal(error); } + @Override public void illegal(String error, Throwable cause) { context.illegal(error, cause); } + @Override public DeployState deployState() { return context.deployState(); } + }; + } + } + + protected abstract void validateManifest(JarContext context, JarFile jar, Manifest mf); + protected abstract void validatePomXml(JarContext context, JarFile jar, Document pom); @Override - public final void validate(VespaModel model, DeployState state) { - ApplicationPackage app = state.getApplicationPackage(); - for (ComponentInfo info : app.getComponentsInfo(state.getVespaVersion())) { + public final void validate(Context context) { + ApplicationPackage app = context.deployState().getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(context.deployState().getVespaVersion())) { Path path = Path.fromString(info.getPathRelativeToAppDir()); try { - state.getDeployLogger() + context.deployState().getDeployLogger() .log(Level.FINE, String.format("Validating bundle at '%s'", path)); JarFile jarFile = new JarFile(app.getFileReference(path)); - validateJarFile(state, jarFile); + validateJarFile(JarContext.of(context), jarFile); } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to validate JAR file '" + path.last() + "'", e); + context.illegal("Failed to validate JAR file '" + path.last() + "'", e); } } } - final void validateJarFile(DeployState state, JarFile jar) throws IOException { + final void validateJarFile(JarContext context, JarFile jar) throws IOException { Manifest manifest = jar.getManifest(); if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename(jar)); + context.illegal("Non-existing or invalid manifest in " + filename(jar)); } - validateManifest(state, jar, manifest); - getPomXmlContent(state.getDeployLogger(), jar) - .ifPresent(pom -> validatePomXml(state, jar, pom)); + validateManifest(context, jar, manifest); + getPomXmlContent(context::illegal, context.deployState().getDeployLogger(), jar).ifPresent(pom -> validatePomXml(context, jar, pom)); } protected final String filename(JarFile jarFile) { return Paths.get(jarFile.getName()).getFileName().toString(); } @@ -89,7 +101,7 @@ public abstract class AbstractBundleValidator extends Validator { } private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); - public Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { + public Optional<Document> getPomXmlContent(BiConsumer<String, Throwable> context, DeployLogger logger, JarFile jar) { return jar.stream() .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) .findFirst() @@ -100,13 +112,13 @@ public abstract class AbstractBundleValidator extends Validator { .parse(new InputSource(new StringReader(text))); } catch (SAXException e) { String message = String.format("Unable to parse pom.xml from %s", filename(jar)); - deployLogger.log(Level.SEVERE, message); - throw new RuntimeException(message, e); + logger.log(Level.SEVERE, message); + context.accept(message, e); } catch (IOException e) { - deployLogger.log(Level.INFO, + logger.log(Level.INFO, String.format("Unable to read '%s' from '%s'", f.getName(), jar.getName())); - return null; } + return null; }); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidator.java index ee37157902c..aee9ca83b08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidator.java @@ -1,9 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CloudName; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Http; @@ -15,29 +14,29 @@ import java.util.logging.Level; * * @author mortent */ -public class AccessControlFilterExcludeValidator extends Validator { +public class AccessControlFilterExcludeValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - if (!deployState.isHosted() || deployState.zone().system().isPublic()) return; - if (deployState.getProperties().allowDisableMtls()) return; - model.getContainerClusters().forEach((id, cluster) -> { + public void validate(Context context) { + if (!context.deployState().isHosted() || context.deployState().zone().system().isPublic()) return; + if (context.deployState().getProperties().allowDisableMtls()) return; + context.model().getContainerClusters().forEach((id, cluster) -> { Http http = cluster.getHttp(); if (http != null) { if (http.getAccessControl().isPresent()) { - verifyNoExclusions(id, http.getAccessControl().get(), deployState); + verifyNoExclusions(id, http.getAccessControl().get(), context); } } }); } - private void verifyNoExclusions(String clusterId, AccessControl accessControl, DeployState deployState) { + private void verifyNoExclusions(String clusterId, AccessControl accessControl, Context context) { if (!accessControl.excludedBindings().isEmpty()) { String message = "Application cluster %s excludes paths from access control, this is not allowed and should be removed.".formatted(clusterId); - if (deployState.zone().cloud().name().equals(CloudName.AWS)) { - throw new IllegalArgumentException(message); + if (context.deployState().zone().cloud().name().equals(CloudName.AWS)) { + context.illegal(message); } else { - deployState.getDeployLogger().log(Level.WARNING, message); + context.deployState().getDeployLogger().log(Level.WARNING, message); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidator.java index 8ea0155dd04..cd6212051cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidator.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Filter; @@ -14,26 +13,26 @@ import com.yahoo.vespa.model.container.http.Http; * * @author bjorncs */ -public class AccessControlFilterValidator extends Validator { +public class AccessControlFilterValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - model.getContainerClusters().forEach((id, cluster) -> { + public void validate(Context context) { + context.model().getContainerClusters().forEach((id, cluster) -> { Http http = cluster.getHttp(); if (http != null) { if (http.getAccessControl().isPresent()) { - verifyAccessControlFilterPresent(http); + verifyAccessControlFilterPresent(context, http); } } }); } - private static void verifyAccessControlFilterPresent(Http http) { + private static void verifyAccessControlFilterPresent(Context context, Http http) { FilterChains filterChains = http.getFilterChains(); Chain<Filter> chain = filterChains.allChains().getComponent(AccessControl.ACCESS_CONTROL_CHAIN_ID); if (chain.getInnerComponents().isEmpty()) { // No access control filter configured - it's up to a config model plugin to provide an implementation of an access control filter. - throw new IllegalArgumentException("The 'access-control' feature is not available in open-source Vespa."); + context.illegal("The 'access-control' feature is not available in open-source Vespa."); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index d877e58e158..0a23e25e432 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -27,10 +27,8 @@ import java.util.regex.Pattern; */ public class BundleValidator extends AbstractBundleValidator { - public BundleValidator() {} - @Override - protected void validateManifest(DeployState state, JarFile jar, Manifest mf) { + protected void validateManifest(JarContext reporter, JarFile jar, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -41,23 +39,22 @@ public class BundleValidator extends AbstractBundleValidator { "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { - throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + filename(jar) + "'"); + reporter.illegal("Required OSGI header '" + header + "' was not found in manifest in '" + filename(jar) + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - log(state, Level.WARNING, + log(reporter.deployState(), Level.WARNING, "Deploying snapshot bundle " + filename(jar) + ".\nTo use this bundle, you must include the " + "qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(state, jar, mf); + validateImportedPackages(reporter.deployState(), jar, mf); } } - @Override protected void validatePomXml(DeployState state, JarFile jar, Document pom) {} + @Override protected void validatePomXml(JarContext reporter, JarFile jar, Document pom) { } private void validateImportedPackages(DeployState state, JarFile jar, Manifest manifest) { Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); @@ -73,7 +70,7 @@ public class BundleValidator extends AbstractBundleValidator { }); deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> { log(state, Level.WARNING, "JAR file '%s' imports the packages %s from '%s'. \n%s", - filename(jar), packagesInUse, artifact.name, artifact.description); + filename(jar), packagesInUse, artifact.name, artifact.description); }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java new file mode 100644 index 00000000000..da96802f864 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import org.bouncycastle.asn1.x509.TBSCertificate; + +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.function.BiConsumer; +import java.util.logging.Level; + +/** + * Validates that trusted data plane certificates are valid + * + * @author bjorncs + */ +public class CloudClientsValidator implements Validator { + + @Override + public void validate(Validation.Context ctx) { + if (!ctx.deployState().isHosted()) return; + ctx.model().getContainerClusters().forEach((clusterName, cluster) -> { + for (var client : cluster.getClients()) { + client.certificates().forEach(cert -> validateCertificate(clusterName, client.id(), cert, ctx::illegal, ctx.deployState())); + } + }); + } + + static void validateCertificate(String clusterName, String clientId, X509Certificate cert, BiConsumer<String, Throwable> reporter, DeployState state) { + try { + var extensions = TBSCertificate.getInstance(cert.getTBSCertificate()).getExtensions(); + if (extensions == null) return; // Certificate without any extensions is okay + if (extensions.getExtensionOIDs().length == 0) { + /* + BouncyCastle 1.77 no longer accepts certificates having an empty sequence of extensions. + Earlier releases violated the ASN.1 specification as the specification forbids empty extension sequence. + See https://github.com/bcgit/bc-java/issues/1479. + + Detect such certificates and issue a warning for now. + Validation will be implicitly enforced once we upgrade BouncyCastle past 1.76. + */ + var message = "The certificate's ASN.1 structure contains an empty sequence of extensions, " + + "which is a violation of the ASN.1 specification. " + + "Please update the application package with a new certificate, " + + "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. " + + "Such certificate will no longer be accepted in near future."; + state.getDeployLogger().log(Level.WARNING, errorMessage(clusterName, clientId, message)); + } + } catch (CertificateEncodingException e) { + reporter.accept(errorMessage(clusterName, clientId, e.getMessage()), e); + } + } + + private static String errorMessage(String clusterName, String clientId, String message) { + return "Client **%s** defined for cluster **%s** contains an invalid certificate: %s" + .formatted(clientId, clusterName, message); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidator.java index 3b50412c44f..d59a76d5804 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidator.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; -import java.io.IOException; import java.security.cert.X509Certificate; import java.util.HashSet; import java.util.List; @@ -18,23 +16,23 @@ import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; -public class CloudDataPlaneFilterValidator extends Validator { +public class CloudDataPlaneFilterValidator implements Validator { private static final Logger log = Logger.getLogger(CloudDataPlaneFilterValidator.class.getName()); @Override - public void validate(VespaModel model, DeployState deployState) { - if (!deployState.isHosted()) return; - if (!deployState.zone().system().isPublic()) return; + public void validate(Context context) { + if (!context.deployState().isHosted()) return; + if (!context.deployState().zone().system().isPublic()) return; - validateUniqueCertificates(deployState); + validateUniqueCertificates(context); } - private void validateUniqueCertificates(DeployState deployState) { - List<NamedReader> certFiles = deployState.getApplicationPackage().getFiles(ApplicationPackage.SECURITY_DIR, ".pem"); + private void validateUniqueCertificates(Context context) { + List<NamedReader> certFiles = context.deployState().getApplicationPackage().getFiles(ApplicationPackage.SECURITY_DIR, ".pem"); Map<String, List<X509Certificate>> configuredCertificates = certFiles.stream() - .collect(Collectors.toMap(NamedReader::getName, CloudDataPlaneFilterValidator::readCertificates)); + .collect(Collectors.toMap(NamedReader::getName, reader -> readCertificates(context, reader))); Set<X509Certificate> duplicates = new HashSet<>(); Set<X509Certificate> globalUniqueCerts = new HashSet<>(); @@ -53,19 +51,21 @@ public class CloudDataPlaneFilterValidator extends Validator { .map(p -> ApplicationPackage.SECURITY_DIR.append(p).getRelative()) .sorted() .toList(); - throw new IllegalArgumentException("Duplicate certificate(s) detected in files: %s. Certificate subject of duplicates: %s" - .formatted(filesWithDuplicates.toString(), - duplicates.stream().map(cert -> cert.getSubjectX500Principal().getName()).toList().toString())); + context.illegal("Duplicate certificate(s) detected in files: %s. Certificate subject of duplicates: %s" + .formatted(filesWithDuplicates.toString(), + duplicates.stream().map(cert -> cert.getSubjectX500Principal().getName()).toList().toString())); } } - private static List<X509Certificate> readCertificates(NamedReader reader) { + private static List<X509Certificate> readCertificates(Context context, NamedReader reader) { try { return X509CertificateUtils.certificateListFromPem(IOUtils.readAll(reader)); - } catch (IOException e) { + } catch (Exception e) { log.warning("Exception reading certificate list from application package. File: %s, exception message: %s" .formatted(reader.getName(), e.getMessage())); - throw new RuntimeException("Error reading certificates from application package", e); + context.illegal("Error reading certificates from application package", e); + return List.of(); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidator.java index 1ddbf4453ae..4e4c8c2916c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidator.java @@ -1,9 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; @@ -16,12 +14,12 @@ import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; * * @author bjorncs */ -public class CloudHttpConnectorValidator extends Validator { +public class CloudHttpConnectorValidator implements Validator { @Override - public void validate(VespaModel model, DeployState state) { - if (!state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + public void validate(Context context) { + if (!context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType())) return; - model.getContainerClusters().forEach((__, cluster) -> { + context.model().getContainerClusters().forEach((__, cluster) -> { var http = cluster.getHttp(); if (http == null) return; var illegalConnectors = http.getHttpServer().stream().flatMap(s -> s.getConnectorFactories().stream() @@ -29,7 +27,7 @@ public class CloudHttpConnectorValidator extends Validator { .map(cf -> "%s@%d".formatted(cf.getName(), cf.getListenPort())) .toList(); if (illegalConnectors.isEmpty()) return; - throw new IllegalArgumentException( + context.illegal( ("Adding additional or modifying existing HTTPS connectors is not allowed for Vespa Cloud applications." + " Violating connectors: %s. See https://cloud.vespa.ai/en/security/whitepaper, " + "https://cloud.vespa.ai/en/security/guide#data-plane.") diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidator.java index 935c3baddd2..0ed49bb96f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidator.java @@ -1,33 +1,34 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.http.HttpFilterChain; -import java.util.Comparator; import java.util.TreeSet; import java.util.logging.Level; import java.util.stream.Collectors; +import static java.util.Comparator.comparing; + /** * Validates that only allowed-listed cloud applications can set up user-specified filter chains * * @author bjorncs */ -public class CloudUserFilterValidator extends Validator { +public class CloudUserFilterValidator implements Validator { @Override - public void validate(VespaModel model, DeployState state) { - if (!state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; - if (state.getProperties().allowUserFilters()) return; - var violations = new TreeSet<Violation>(); - for (var cluster : model.getContainerClusters().values()) { + public void validate(Context context) { + if (!context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType())) return; + if (context.deployState().getProperties().allowUserFilters()) return; + record Violation(String cluster, String chain) { } + var violations = new TreeSet<Violation>(comparing(Violation::chain).thenComparing(Violation::cluster)); + for (var cluster : context.model().getContainerClusters().values()) { if (cluster.getHttp() == null) continue; for (var chain : cluster.getHttp().getFilterChains().allChains().allComponents()) { if (chain.type() == HttpFilterChain.Type.USER) { var msg = "Found filter chain violation - chain '%s' in cluster '%s'".formatted(cluster.name(), chain.id()); - state.getDeployLogger().log(Level.WARNING, msg); + context.deployState().getDeployLogger().log(Level.WARNING, msg); violations.add(new Violation(cluster.name(), chain.id())); } } @@ -37,14 +38,7 @@ public class CloudUserFilterValidator extends Validator { .map(v -> "chain '%s' in cluster '%s'".formatted(v.chain(), v.cluster())) .collect(Collectors.joining(", ", "[", "]")); var msg = ("HTTP filter chains are currently not supported in Vespa Cloud (%s)").formatted(violationsStr); - throw new IllegalArgumentException(msg); - } - - private record Violation(String cluster, String chain) implements Comparable<Violation> { - @Override - public int compareTo(Violation other) { - return Comparator.comparing(Violation::chain).thenComparing(Violation::cluster).compare(this, other); - } + context.illegal(msg); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java index c7ae8f4f4a3..12f3f025996 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java @@ -2,19 +2,17 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.ComplexAttributeFieldUtils; import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.document.ImmutableSDField; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.SearchCluster; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -24,24 +22,24 @@ import java.util.stream.Collectors; * * @author geirst */ -public class ComplexFieldsWithStructFieldAttributesValidator extends Validator { +public class ComplexFieldsWithStructFieldAttributesValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - List<SearchCluster> searchClusters = model.getSearchClusters(); + public void validate(Context context) { + List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { if (cluster.isStreaming()) continue; for (SchemaInfo spec : cluster.schemas().values()) { - validateComplexFields(cluster.getClusterName(), spec.fullSchema(), deployState.getDeployLogger()); + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger()); } } } - private static void validateComplexFields(String clusterName, Schema schema, DeployLogger logger) { + private static void validateComplexFields(Context context, String clusterName, Schema schema, DeployLogger logger) { String unsupportedFields = validateComplexFields(schema); if (!unsupportedFields.isEmpty()) { - throw new IllegalArgumentException(getErrorMessage(clusterName, schema, unsupportedFields)); + context.illegal(getErrorMessage(clusterName, schema, unsupportedFields)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java index b969387724c..3dbba081400 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java @@ -1,11 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.schema.Schema; import com.yahoo.schema.document.ImmutableSDField; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.util.ArrayList; import java.util.Collection; @@ -21,21 +19,21 @@ import java.util.stream.Collectors; * * @author geirst */ -public class ComplexFieldsWithStructFieldIndexesValidator extends Validator { +public class ComplexFieldsWithStructFieldIndexesValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - for (var cluster : model.getSearchClusters()) { + public void validate(Context context) { + for (var cluster : context.model().getSearchClusters()) { if (cluster.isStreaming()) { continue; } for (var spec : cluster.schemas().values()) { - validateComplexFields(cluster.getClusterName(), spec.fullSchema(), deployState.getDeployLogger()); + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema()); } } } - private static void validateComplexFields(String clusterName, Schema schema, DeployLogger logger) { + private static void validateComplexFields(Context context, String clusterName, Schema schema) { String unsupportedFields = schema.allFields() .filter(field -> hasStructFieldsWithIndex(field)) .map(ComplexFieldsWithStructFieldIndexesValidator::toString) @@ -43,7 +41,8 @@ public class ComplexFieldsWithStructFieldIndexesValidator extends Validator { if (!unsupportedFields.isEmpty()) { // TODO (Vespa 9 or before): Change back to an exception when no applications are using it wrong. - logger.logApplicationPackage(Level.WARNING, + context.deployState().getDeployLogger().logApplicationPackage( + Level.WARNING, String.format("For cluster '%s', schema '%s': The following complex fields have struct fields with 'indexing: index' which is not supported and has no effect: %s. " + "Remove setting or change to 'indexing: attribute' if needed for matching.", clusterName, schema.getName(), unsupportedFields)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java index e4a07622ea3..b3802bf211b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.schema.DistributableResource; import com.yahoo.schema.RankProfile; import com.yahoo.schema.Schema; -import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.io.FileNotFoundException; @@ -19,22 +18,22 @@ import java.io.FileNotFoundException; * * @author Vegard Sjonfjell */ -public class ConstantValidator extends Validator { +public class ConstantValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { + public void validate(Context context) { var exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - for (Schema schema : deployState.getSchemas()) { + for (Schema schema : context.deployState().getSchemas()) { for (var constant : schema.declaredConstants().values()) - validate(constant, deployState.getApplicationPackage(), exceptionMessageCollector); - for (var profile : deployState.rankProfileRegistry().rankProfilesOf(schema)) { + validate(constant, context.deployState().getApplicationPackage(), exceptionMessageCollector); + for (var profile : context.deployState().rankProfileRegistry().rankProfilesOf(schema)) { for (var constant : profile.declaredConstants().values()) - validate(constant, deployState.getApplicationPackage(), exceptionMessageCollector); + validate(constant, context.deployState().getApplicationPackage(), exceptionMessageCollector); } } if (exceptionMessageCollector.exceptionsOccurred) - throw new IllegalArgumentException(exceptionMessageCollector.combinedMessage); + context.illegal(exceptionMessageCollector.combinedMessage); } private void validate(RankProfile.Constant constant, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java index 49ff9b4cfde..71b5b47b732 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java @@ -1,20 +1,19 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; /** * Validates that a Vespa Cloud application has at least one container cluster. * * @author jonmv */ -public class ContainerInCloudValidator extends Validator { +public class ContainerInCloudValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - if (deployState.isHosted() && model.getContainerClusters().isEmpty()) - throw new IllegalArgumentException("Vespa Cloud applications must have at least one container cluster"); + public void validate(Context context) { + if (context.deployState().isHosted() && context.model().getContainerClusters().isEmpty()) + context.illegal("Vespa Cloud applications must have at least one container cluster"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java index 7e0df4cf1fa..1675bacb387 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java @@ -3,9 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.ContainerModel; import java.io.Reader; @@ -18,29 +17,29 @@ import java.util.Optional; * @author hmusum * @author bratseth */ -public class DeploymentSpecValidator extends Validator { +public class DeploymentSpecValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); + public void validate(Context context) { + Optional<Reader> deployment = context.deployState().getApplicationPackage().getDeployment(); if ( deployment.isEmpty()) return; Reader deploymentReader = deployment.get(); DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); - List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class); + List<ContainerModel> containers = context.model().getRoot().configModelRepo().getModels(ContainerModel.class); for (DeploymentInstanceSpec instance : deploymentSpec.instances()) { instance.endpoints().forEach(endpoint -> { - requireClusterId(containers, instance.name(), "Endpoint '" + endpoint.endpointId() + "'", - endpoint.containerId()); + requireClusterId(context, containers, instance.name(), + "Endpoint '" + endpoint.endpointId() + "'", endpoint.containerId()); }); } } - private static void requireClusterId(List<ContainerModel> containers, InstanceName instanceName, String context, - String id) { + private static void requireClusterId(Context context, List<ContainerModel> containers, InstanceName instanceName, + String endpoint, String id) { if (containers.stream().noneMatch(container -> container.getCluster().getName().equals(id))) - throw new IllegalArgumentException(context + " in instance " + instanceName + ": '" + id + - "' specified in deployment.xml does not match any container cluster ID"); + context.illegal(endpoint + " in instance " + instanceName + ": '" + id + + "' specified in deployment.xml does not match any container cluster ID"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index 635f7c67dd6..f2887d9f9b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -1,18 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; -public class EndpointCertificateSecretsValidator extends Validator { +public class EndpointCertificateSecretsValidator implements Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override - public void validate(VespaModel model, DeployState deployState) { - if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get().isMissing()) { + public void validate(Context context) { + if (context.deployState().endpointCertificateSecrets().isPresent() && context.deployState().endpointCertificateSecrets().get().isMissing()) { throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate version %s for application %s" - .formatted(deployState.endpointCertificateSecrets().get().version(), deployState.getProperties().applicationId().serializedForm())); + .formatted(context.deployState().endpointCertificateSecrets().get().version(), context.deployState().getProperties().applicationId().serializedForm())); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java index 30209d0bdee..d4f26530b75 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java @@ -2,9 +2,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.util.logging.Logger; @@ -13,18 +12,19 @@ import java.util.logging.Logger; * * @author mortent */ -public class InfrastructureDeploymentValidator extends Validator { +public class InfrastructureDeploymentValidator implements Validator { private static final Logger log = Logger.getLogger(InfrastructureDeploymentValidator.class.getName()); @Override - public void validate(VespaModel model, DeployState deployState) { + public void validate(Context context) { // Allow the internally defined tenant owning all infrastructure applications - if (ApplicationId.global().tenant().equals(model.applicationPackage().getApplicationId().tenant())) return; - ConfigModelContext.ApplicationType applicationType = model.getAdmin().getApplicationType(); + if (TenantName.from("hosted-vespa").equals(context.model().applicationPackage().getApplicationId().tenant())) return; + ConfigModelContext.ApplicationType applicationType = context.model().getAdmin().getApplicationType(); if (applicationType != ConfigModelContext.ApplicationType.DEFAULT) { - log.warning("Tenant %s is not allowed to use application type %s".formatted(model.applicationPackage().getApplicationId().toFullString(), applicationType)); - throw new IllegalArgumentException("Tenant is not allowed to override application type"); + log.warning("Tenant %s is not allowed to use application type %s".formatted(context.model().applicationPackage().getApplicationId().toFullString(), applicationType)); + context.illegal("Tenant is not allowed to override application type"); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index e9038ff2b0f..482c4477cdc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -1,10 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.Text; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.util.logging.Level; @@ -13,37 +11,37 @@ import java.util.logging.Level; * * @author bjorncs */ -public class JvmHeapSizeValidator extends Validator { +public class JvmHeapSizeValidator implements Validator { public static final int percentLimit = 15; public static final double gbLimit = 0.6; @Override - public void validate(VespaModel model, DeployState ds) { - if (!ds.featureFlags().dynamicHeapSize()) return; - if (!ds.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + public void validate(Context context) { + if (!context.deployState().featureFlags().dynamicHeapSize()) return; + if (!context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType())) return; - model.getContainerClusters().forEach((clusterId, appCluster) -> { + context.model().getContainerClusters().forEach((clusterId, appCluster) -> { var mp = appCluster.getMemoryPercentage().orElse(null); if (mp == null) return; if (mp.availableMemoryGb().isEmpty()) { - ds.getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); + context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); return; } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); - ds.getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", + context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); if (mp.percentage() < percentLimit) { - throw new IllegalArgumentException(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", clusterId, mp.percentage(), percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { - throw new IllegalArgumentException( + context.illegal( Text.format("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 15d3e63c7fa..15d293e4abc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -1,16 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.schema.Index; import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchAlgorithm; -import com.yahoo.schema.Index; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import java.util.Map; @@ -19,11 +18,11 @@ import java.util.Map; * * @author vegardh */ -public class NoPrefixForIndexes extends Validator { +public class NoPrefixForIndexes implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - for (SearchCluster cluster : model.getSearchClusters()) { + public void validate(Context context) { + for (SearchCluster cluster : context.model().getSearchClusters()) { if (cluster instanceof IndexedSearchCluster) { IndexedSearchCluster sc = (IndexedSearchCluster) cluster; for (DocumentDatabase docDb : sc.getDocumentDbs()) { @@ -33,11 +32,11 @@ public class NoPrefixForIndexes extends Validator { if (field.doesIndexing()) { //if (!field.getIndexTo().isEmpty() && !field.getIndexTo().contains(field.getName())) continue; if (field.getMatching().getAlgorithm().equals(MatchAlgorithm.PREFIX)) { - failField(schema, field); + failField(context, schema, field); } for (Map.Entry<String, Index> e : field.getIndices().entrySet()) { if (e.getValue().isPrefix()) { - failField(schema, field); + failField(context, schema, field); } } } @@ -47,8 +46,8 @@ public class NoPrefixForIndexes extends Validator { } } - private void failField(Schema schema, ImmutableSDField field) { - throw new IllegalArgumentException("For " + schema + ", field '" + field.getName() + - "': match/index:prefix is not supported for indexes."); + private void failField(Context context, Schema schema, ImmutableSDField field) { + context.illegal("For " + schema + ", field '" + field.getName() + + "': match/index:prefix is not supported for indexes."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java index 68e0172931a..e0a43f0988a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import org.w3c.dom.Document; import java.util.Arrays; @@ -15,17 +14,17 @@ import java.util.logging.Level; public class PublicApiBundleValidator extends AbstractBundleValidator { @Override - protected void validateManifest(DeployState state, JarFile jar, Manifest mf) { + protected void validateManifest(JarContext context, JarFile jar, Manifest mf) { String nonPublicApiAttribute = mf.getMainAttributes().getValue("X-JDisc-Non-PublicApi-Import-Package"); if (nonPublicApiAttribute == null) return; var nonPublicApisUsed = Arrays.asList(nonPublicApiAttribute.split(",")); if (! nonPublicApisUsed.isEmpty()) { - log(state, Level.WARNING, "Jar file '%s' uses non-public Vespa APIs: %s", filename(jar), nonPublicApisUsed); + log(context.deployState(), Level.WARNING, "Jar file '%s' uses non-public Vespa APIs: %s", filename(jar), nonPublicApisUsed); } } @Override - protected void validatePomXml(DeployState state, JarFile jar, Document pom) { } + protected void validatePomXml(JarContext context, JarFile jar, Document pom) { } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 86cedd3ebbf..4d9386b5f19 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -10,6 +9,7 @@ import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.math.BigDecimal; import java.util.LinkedHashSet; @@ -24,16 +24,16 @@ import java.util.stream.Collectors; * * @author ogronnesby */ -public class QuotaValidator extends Validator { +public class QuotaValidator implements Validator { private static final Logger log = Logger.getLogger(QuotaValidator.class.getName()); private static final Capacity zeroCapacity = Capacity.from(new ClusterResources(0, 0, NodeResources.zero())); @Override - public void validate(VespaModel model, DeployState deployState) { - var quota = deployState.getProperties().quota(); - quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, model)); - quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, model, deployState.getProperties().zone())); + public void validate(Context context) { + var quota = context.deployState().getProperties().quota(); + quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, context.model())); + quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context.model(), context.deployState().getProperties().zone())); } private void validateBudget(BigDecimal budget, VespaModel model, Zone zone) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b3fb25be2e5..ebfce1353d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -5,7 +5,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.collections.Pair; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; @@ -21,11 +20,12 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.yolean.Exceptions; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -44,7 +44,7 @@ import java.util.logging.Logger; * * @author vegardh */ -public class RankSetupValidator extends Validator { +public class RankSetupValidator implements Validator { private static final Logger log = Logger.getLogger(RankSetupValidator.class.getName()); private static final String binaryName = "vespa-verify-ranksetup-bin "; @@ -56,14 +56,14 @@ public class RankSetupValidator extends Validator { } @Override - public void validate(VespaModel model, DeployState deployState) { + public void validate(Context context) { File cfgDir = null; try { cfgDir = Files.createTempDirectory("verify-ranksetup." + - deployState.getProperties().applicationId().toFullString() + + context.deployState().getProperties().applicationId().toFullString() + ".") .toFile(); - for (SearchCluster cluster : model.getSearchClusters()) { + for (SearchCluster cluster : context.model().getSearchClusters()) { // Skipping ranking expression checking for streaming clusters, not implemented yet if (cluster.isStreaming()) continue; @@ -74,24 +74,24 @@ public class RankSetupValidator extends Validator { String schemaDir = clusterDir + schemaName + "/"; writeConfigs(schemaDir, docDb); writeExtraVerifyRankSetupConfig(schemaDir, docDb); - if (!validate("dir:" + schemaDir, sc, schemaName, deployState.getDeployLogger(), cfgDir)) { + if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir)) { return; } } } } catch (IOException e) { - throw new RuntimeException(e); + context.illegal("unable to read rank setup", e); } finally { if (cfgDir != null) deleteTempDir(cfgDir); } } - private boolean validate(String configId, SearchCluster searchCluster, String schema, DeployLogger deployLogger, File tempDir) { + private boolean validate(Context context, String configId, SearchCluster searchCluster, String schema, File tempDir) { Instant start = Instant.now(); try { log.log(Level.FINE, () -> String.format("Validating schema '%s' for cluster %s with config id %s", schema, searchCluster, configId)); - boolean ret = execValidate(configId, searchCluster, schema, deployLogger); + boolean ret = execValidate(context, configId, searchCluster, schema); if (!ret) { // Give up, don't log same error msg repeatedly deleteTempDir(tempDir); @@ -100,7 +100,8 @@ public class RankSetupValidator extends Validator { return ret; } catch (IllegalArgumentException e) { deleteTempDir(tempDir); - throw e; + context.illegal("failed validating rank setup", e); + return false; } } @@ -170,17 +171,17 @@ public class RankSetupValidator extends Validator { IOUtils.writeFile(dir + configName, StringUtilities.implodeMultiline(ConfigInstance.serialize(config)), false); } - private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { + private boolean execValidate(Context context, String configId, SearchCluster sc, String sdName) { String command = String.format("%s %s", binaryName, configId); try { Pair<Integer, String> ret = new ProcessExecuter(true).exec(command); Integer exitCode = ret.getFirst(); String output = ret.getSecond(); if (exitCode != 0) { - validateFail(output, exitCode, sc, sdName, deployLogger); + validateFail(context, output, exitCode, sc, sdName); } } catch (IOException e) { - validateWarn(e, deployLogger); + validateWarn(e, context.deployState().getDeployLogger()); return false; } return true; @@ -193,7 +194,7 @@ public class RankSetupValidator extends Validator { deployLogger.logApplicationPackage(Level.WARNING, msg); } - private void validateFail(String output, int exitCode, SearchCluster sc, String sdName, DeployLogger deployLogger) { + private void validateFail(Context context, String output, int exitCode, SearchCluster sc, String sdName) { StringBuilder message = new StringBuilder("Error in rank setup in schema '").append(sdName) .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n"); if (output.isEmpty()) { @@ -224,9 +225,9 @@ public class RankSetupValidator extends Validator { } if (ignoreValidationErrors) { - deployLogger.log(Level.WARNING, message.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); + context.deployState().getDeployLogger().log(Level.WARNING, message.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { - throw new IllegalArgumentException(message.toString()); + context.illegal(message.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java index d2a26d87899..bd933f1c656 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java @@ -1,21 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.select.DocumentSelector; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; /** * Validates routing selector for search and content clusters */ -public class RoutingSelectorValidator extends Validator { +public class RoutingSelectorValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - for (SearchCluster cluster : model.getSearchClusters()) { + public void validate(Context context) { + for (SearchCluster cluster : context.model().getSearchClusters()) { if (cluster instanceof IndexedSearchCluster) { IndexedSearchCluster sc = (IndexedSearchCluster) cluster; String routingSelector = sc.getRoutingSelector(); @@ -23,8 +22,7 @@ public class RoutingSelectorValidator extends Validator { try { new DocumentSelector(routingSelector); } catch (com.yahoo.document.select.parser.ParseException e) { - throw new IllegalArgumentException("Failed to parse routing selector for search cluster '" + - sc.getClusterName() + "'", e); + context.illegal("Failed to parse routing selector for search cluster '" + sc.getClusterName() + "'", e); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java index 25cc2596fb2..87b5e8c8fc4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java @@ -1,27 +1,25 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.util.List; /** * Validates routing - * */ -public class RoutingValidator extends Validator { +public class RoutingValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - List<String> errors = model.getRouting().getErrors(); + public void validate(Context context) { + List<String> errors = context.model().getRouting().getErrors(); if (!errors.isEmpty()) { StringBuilder msg = new StringBuilder(); msg.append("The routing specification contains ").append(errors.size()).append(" error(s):\n"); for (int i = 0, len = errors.size(); i < len; ++i) { msg.append(i + 1).append(". ").append(errors.get(i)).append("\n"); } - throw new IllegalArgumentException(msg.toString()); + context.illegal(msg.toString()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java index 6827cba4030..b9278dbc08d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java @@ -3,8 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.util.logging.Level; @@ -13,17 +12,14 @@ import java.util.logging.Level; * * @author hmusum */ -public class SchemasDirValidator extends Validator { - - public SchemasDirValidator() { - } +public class SchemasDirValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage app = deployState.getApplicationPackage(); + public void validate(Context context) { + ApplicationPackage app = context.deployState().getApplicationPackage(); ApplicationFile sdDir = app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR); if (sdDir.exists() && sdDir.isDirectory()) - deployState.getDeployLogger().logApplicationPackage( + context.deployState().getDeployLogger().logApplicationPackage( Level.WARNING, "Directory " + ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative() + "/ should not be used for schemas, use " + ApplicationPackage.SCHEMAS_DIR.getRelative() + "/ instead"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 51f58ea5f88..6e21adc4fe4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -1,21 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.SearchCluster; import java.util.List; @@ -26,11 +25,11 @@ import java.util.List; * * @author Simon Thoresen Hult */ -public class SearchDataTypeValidator extends Validator { +public class SearchDataTypeValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - List<SearchCluster> clusters = model.getSearchClusters(); + public void validate(Context context) { + List<SearchCluster> clusters = context.model().getSearchClusters(); for (SearchCluster cluster : clusters) { if (cluster.isStreaming()) { continue; @@ -40,22 +39,22 @@ public class SearchDataTypeValidator extends Validator { if (docType == null) { continue; } - validateDocument(cluster, spec.fullSchema(), docType); + validateDocument(context, cluster, spec.fullSchema(), docType); } } } - private void validateDocument(SearchCluster cluster, Schema schema, SDDocumentType doc) { + private void validateDocument(Context context, SearchCluster cluster, Schema schema, SDDocumentType doc) { for (SDDocumentType child : doc.getTypes()) { - validateDocument(cluster, schema, child); + validateDocument(context, cluster, schema, child); } for (Field field : doc.fieldSet()) { DataType fieldType = field.getDataType(); - disallowIndexingOfMaps(cluster, schema, field); + disallowIndexingOfMaps(context, cluster, schema, field); if ( ! isSupportedInSearchClusters(fieldType)) { - throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " + - "clusters (field '" + field.getName() + "' in schema '" + - schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + context.illegal("Field type '" + fieldType.getName() + "' is illegal for search " + + "clusters (field '" + field.getName() + "' in schema '" + + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } } @@ -85,12 +84,13 @@ public class SearchDataTypeValidator extends Validator { } } - private void disallowIndexingOfMaps(SearchCluster cluster, Schema schema, Field field) { + private void disallowIndexingOfMaps(Context context, SearchCluster cluster, Schema schema, Field field) { DataType fieldType = field.getDataType(); if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { - throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + - "clusters (field '" + field.getName() + "' in definition '" + - schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + context.illegal("Field type '" + fieldType.getName() + "' cannot be indexed for search " + + "clusters (field '" + field.getName() + "' in definition '" + + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java index 9c87415395b..afa29533b93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SecretStoreValidator.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.ConfigModelContext.ApplicationType; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.IdentityProvider; @@ -14,18 +13,18 @@ import com.yahoo.vespa.model.container.component.Component; * * @author gjoranv */ -public class SecretStoreValidator extends Validator { +public class SecretStoreValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - if (! deployState.isHosted()) return; - if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return; + public void validate(Context context) { + if (! context.deployState().isHosted()) return; + if (context.model().getAdmin().getApplicationType() != ApplicationType.DEFAULT) return; - for (ContainerCluster cluster : model.getContainerClusters().values()) { + for (ContainerCluster<?> cluster : context.model().getContainerClusters().values()) { if (cluster.getSecretStore().isPresent() && ! hasIdentityProvider(cluster)) - throw new IllegalArgumentException(String.format( - "Container cluster '%s' uses a secret store, so an Athenz domain and an Athenz service" + - " must be declared in deployment.xml.", cluster.getName())); + context.illegal(String.format( + "Container cluster '%s' uses a secret store, so an Athenz domain and an Athenz service" + + " must be declared in deployment.xml.", cluster.getName())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 42dc4df0d43..c0ad55fc8f4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.TensorDataType; @@ -10,7 +9,7 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchType; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.StreamingSearchCluster; @@ -20,18 +19,18 @@ import java.util.logging.Level; /** * Validates streaming mode */ -public class StreamingValidator extends Validator { +public class StreamingValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - List<SearchCluster> searchClusters = model.getSearchClusters(); + public void validate(Context context) { + List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { if ( ! cluster.isStreaming()) continue; var streamingCluster = (StreamingSearchCluster)cluster; - warnStreamingAttributes(streamingCluster, deployState.getDeployLogger()); - warnStreamingGramMatching(streamingCluster, deployState.getDeployLogger()); - failStreamingDocumentReferences(streamingCluster); + warnStreamingAttributes(streamingCluster, context.deployState().getDeployLogger()); + warnStreamingGramMatching(streamingCluster, context.deployState().getDeployLogger()); + failStreamingDocumentReferences(context, streamingCluster); } } @@ -81,14 +80,14 @@ public class StreamingValidator extends Validator { "': 'attribute' has same match semantics as 'index'."); } - private static void failStreamingDocumentReferences(StreamingSearchCluster sc) { + private static void failStreamingDocumentReferences(Context context, StreamingSearchCluster sc) { for (Attribute attribute : sc.derived().getAttributeFields().attributes()) { DataType dataType = attribute.getDataType(); if (dataType instanceof NewDocumentReferenceDataType) { String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " + "Document references and imported fields are not allowed in streaming search.", sc.getClusterName(), attribute.getName(), dataType.getName()); - throw new IllegalArgumentException(errorMessage); + context.illegal(errorMessage); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java index ed53aa581b1..3f519088a56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Handler; @@ -18,70 +17,70 @@ import java.util.logging.Level; * * @author bjorncs */ -class UriBindingsValidator extends Validator { +class UriBindingsValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { + public void validate(Context context) { + for (ApplicationContainerCluster cluster : context.model().getContainerClusters().values()) { for (Handler handler : cluster.getHandlers()) { for (BindingPattern binding : handler.getServerBindings()) { - validateUserBinding(binding, model, deployState); + validateUserBinding(binding, context); } } Http http = cluster.getHttp(); if (http != null) { for (FilterBinding binding : cluster.getHttp().getBindings()) { - validateUserBinding(binding.binding(), model, deployState); + validateUserBinding(binding.binding(), context); } } } } - private static void validateUserBinding(BindingPattern binding, VespaModel model, DeployState deployState) { - validateScheme(binding, deployState); - if (isHostedApplication(model, deployState)) { - validateHostedApplicationUserBinding(binding, deployState); + private static void validateUserBinding(BindingPattern binding, Context context) { + validateScheme(binding, context); + if (isHostedApplication(context)) { + validateHostedApplicationUserBinding(binding, context); } } - private static void validateScheme(BindingPattern binding, DeployState deployState) { + private static void validateScheme(BindingPattern binding, Context context) { if (binding.scheme().equals("https")) { String message = createErrorMessage( binding, "'https' bindings are deprecated, use 'http' instead to bind to both http and https traffic."); - deployState.getDeployLogger().logApplicationPackage(Level.WARNING, message); + context.deployState().getDeployLogger().logApplicationPackage(Level.WARNING, message); } } - private static void validateHostedApplicationUserBinding(BindingPattern binding, DeployState deployState) { + private static void validateHostedApplicationUserBinding(BindingPattern binding, Context context) { // only perform these validation for used-generated bindings // bindings produced by the hosted config model amender will violate some of the rules below if (binding instanceof SystemBindingPattern) return; // Allow binding to port if we are restricting data plane bindings if (!binding.matchesAnyPort()) { - logOrThrow(createErrorMessage(binding, "binding with port is not allowed"), deployState); + logOrThrow(createErrorMessage(binding, "binding with port is not allowed"), context); } if (!binding.host().equals(BindingPattern.WILDCARD_PATTERN)) { - logOrThrow(createErrorMessage(binding, "only binding with wildcard ('*') for hostname is allowed"), deployState); + logOrThrow(createErrorMessage(binding, "only binding with wildcard ('*') for hostname is allowed"), context); } if (!binding.scheme().equals("http") && !binding.scheme().equals("https")) { - logOrThrow(createErrorMessage(binding, "only 'http' is allowed as scheme"), deployState); + logOrThrow(createErrorMessage(binding, "only 'http' is allowed as scheme"), context); } } /* * Logs to deploy logger in non-public systems, throw otherwise */ - private static void logOrThrow(String message, DeployState deployState) { - if (deployState.zone().system().isPublic()) { - throw new IllegalArgumentException(message); + private static void logOrThrow(String message, Context context) { + if (context.deployState().zone().system().isPublic()) { + context.illegal(message); } else { - deployState.getDeployLogger().log(Level.WARNING, message); + context.deployState().getDeployLogger().log(Level.WARNING, message); } } - private static boolean isHostedApplication(VespaModel model, DeployState deployState) { - return deployState.isHostedTenantApplication(model.getAdmin().getApplicationType()); + private static boolean isHostedApplication(Context context) { + return context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType()); } private static String createErrorMessage(BindingPattern binding, String message) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java index 5db88dbd0db..5619e99308d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -1,8 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.container.ApplicationContainerCluster; /** @@ -10,15 +10,15 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; * * @author hmusum */ -public class UrlConfigValidator extends Validator { +public class UrlConfigValidator implements Validator { @Override - public void validate(VespaModel model, DeployState state) { - if (! state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + public void validate(Context context) { + if (! context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType())) return; - model.getContainerClusters().forEach((__, cluster) -> { - var isExclusive = hasExclusiveNodes(model, cluster); - validateS3UlsInConfig(state, cluster, isExclusive); + context.model().getContainerClusters().forEach((__, cluster) -> { + var isExclusive = hasExclusiveNodes(context.model(), cluster); + validateS3UlsInConfig(context, cluster, isExclusive); }); } @@ -30,15 +30,15 @@ public class UrlConfigValidator extends Validator { .anyMatch(membership -> membership.cluster().isExclusive()); } - private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { + private static void validateS3UlsInConfig(Context context, ApplicationContainerCluster cluster, boolean isExclusive) { if (hasS3UrlInConfig(cluster)) { // TODO: Would be even better if we could add which config/field the url is set for in the error message String message = "Found s3:// urls in config for container cluster " + cluster.getName(); - if ( ! state.zone().system().isPublic()) - throw new IllegalArgumentException(message + ". This is only supported in public systems"); + if ( ! context.deployState().zone().system().isPublic()) + context.illegal(message + ". This is only supported in public systems"); else if ( ! isExclusive) - throw new IllegalArgumentException(message + ". Nodes in the cluster need to be 'exclusive'," + - " see https://cloud.vespa.ai/en/reference/services#nodes"); + context.illegal(message + ". Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 0f7a415c33a..dc7a2651e1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -10,7 +10,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.CertificateRemovalChangeValidator; -import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator; import com.yahoo.vespa.model.application.validation.change.ContainerRestartValidator; import com.yahoo.vespa.model.application.validation.change.ContentClusterRemovalValidator; @@ -32,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -65,14 +65,16 @@ public class Validation { validateModel(validationParameters, execution); - additionalValidators.forEach(execution::run); + for (Validator validator : additionalValidators) { + validator.validate(execution); + } if (deployState.getProperties().isFirstTimeDeployment()) { validateFirstTimeDeployment(execution); } - else if (deployState.getPreviousModel().isPresent() && (deployState.getPreviousModel().get() instanceof VespaModel vespaModel)) { - validateChanges(vespaModel, execution); - // TODO: Why is this done here? It won't be done on more than one config server? + else if (deployState.getPreviousModel().isPresent() && (deployState.getPreviousModel().get() instanceof VespaModel)) { + validateChanges(execution); + // TODO: Why is this done here? It won't be done on more than one config server? deferConfigChangesForClustersToBeRestarted(execution.actions, model); } @@ -81,57 +83,59 @@ public class Validation { } private static void validateRouting(Execution execution) { - execution.run(new RoutingValidator()); - execution.run(new RoutingSelectorValidator()); + new RoutingValidator().validate(execution); + new RoutingSelectorValidator().validate(execution); } private static void validateModel(ValidationParameters validationParameters, Execution execution) { - execution.run(new SchemasDirValidator()); - execution.run(new BundleValidator()); - execution.run(new PublicApiBundleValidator()); - execution.run(new SearchDataTypeValidator()); - execution.run(new ComplexFieldsWithStructFieldAttributesValidator()); - execution.run(new ComplexFieldsWithStructFieldIndexesValidator()); - execution.run(new StreamingValidator()); - execution.run(new RankSetupValidator(validationParameters.ignoreValidationErrors())); - execution.run(new NoPrefixForIndexes()); - execution.run(new ContainerInCloudValidator()); - execution.run(new DeploymentSpecValidator()); - execution.run(new ValidationOverridesValidator()); - execution.run(new ConstantValidator()); - execution.run(new SecretStoreValidator()); - execution.run(new EndpointCertificateSecretsValidator()); - execution.run(new AccessControlFilterValidator()); - execution.run(new QuotaValidator()); - execution.run(new UriBindingsValidator()); - execution.run(new CloudDataPlaneFilterValidator()); - execution.run(new AccessControlFilterExcludeValidator()); - execution.run(new CloudUserFilterValidator()); - execution.run(new CloudHttpConnectorValidator()); - execution.run(new UrlConfigValidator()); - execution.run(new JvmHeapSizeValidator()); + new SchemasDirValidator().validate(execution); + new BundleValidator().validate(execution); + new PublicApiBundleValidator().validate(execution); + new SearchDataTypeValidator().validate(execution); + new ComplexFieldsWithStructFieldAttributesValidator().validate(execution); + new ComplexFieldsWithStructFieldIndexesValidator().validate(execution); + new StreamingValidator().validate(execution); + new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(execution); + new NoPrefixForIndexes().validate(execution); + new ContainerInCloudValidator().validate(execution); + new DeploymentSpecValidator().validate(execution); + new ValidationOverridesValidator().validate(execution); + new ConstantValidator().validate(execution); + new SecretStoreValidator().validate(execution); + new AccessControlFilterValidator().validate(execution); + new QuotaValidator().validate(execution); + new UriBindingsValidator().validate(execution); + new CloudDataPlaneFilterValidator().validate(execution); + new AccessControlFilterExcludeValidator().validate(execution); + new CloudUserFilterValidator().validate(execution); + new CloudHttpConnectorValidator().validate(execution); + new UrlConfigValidator().validate(execution); + new JvmHeapSizeValidator().validate(execution); + new InfrastructureDeploymentValidator().validate(execution); + new EndpointCertificateSecretsValidator().validate(execution); + new CloudClientsValidator().validate(execution); } private static void validateFirstTimeDeployment(Execution execution) { - execution.run(new RedundancyValidator()); + new RedundancyValidator().validate((Context) execution); } - private static void validateChanges(VespaModel currentModel, Execution execution) { - execution.run(new IndexingModeChangeValidator(), currentModel); - execution.run(new GlobalDocumentChangeValidator(), currentModel); - execution.run(new IndexedSearchClusterChangeValidator(), currentModel); - execution.run(new StreamingSearchClusterChangeValidator(), currentModel); - execution.run(new ConfigValueChangeValidator(), currentModel); - execution.run(new StartupCommandChangeValidator(), currentModel); - execution.run(new ContentTypeRemovalValidator(), currentModel); - execution.run(new ContentClusterRemovalValidator(), currentModel); - execution.run(new ResourcesReductionValidator(), currentModel); - execution.run(new ContainerRestartValidator(), currentModel); - execution.run(new NodeResourceChangeValidator(), currentModel); - execution.run(new RedundancyIncreaseValidator(), currentModel); - execution.run(new CertificateRemovalChangeValidator(), currentModel); - execution.run(new RedundancyValidator(), currentModel); - execution.run(new RestartOnDeployForOnnxModelChangesValidator(), currentModel); + private static void validateChanges(Execution execution) { + new IndexingModeChangeValidator().validate(execution); + new GlobalDocumentChangeValidator().validate(execution); + new IndexedSearchClusterChangeValidator().validate(execution); + new StreamingSearchClusterChangeValidator().validate(execution); + new ConfigValueChangeValidator().validate(execution); + new StartupCommandChangeValidator().validate(execution); + new ContentTypeRemovalValidator().validate(execution); + new ContentClusterRemovalValidator().validate(execution); + new ResourcesReductionValidator().validate(execution); + new ContainerRestartValidator().validate(execution); + new NodeResourceChangeValidator().validate(execution); + new RedundancyIncreaseValidator().validate(execution); + new CertificateRemovalChangeValidator().validate(execution); + new RedundancyValidator().validate(execution); + new RestartOnDeployForOnnxModelChangesValidator().validate(execution); } private static void deferConfigChangesForClustersToBeRestarted(List<ConfigChangeAction> actions, VespaModel model) { @@ -150,59 +154,101 @@ public class Validation { } } + public interface Context { + /** Auxiliary deploy state of the application. */ + DeployState deployState(); + /** The model to validate. */ + VespaModel model(); + /** Report a failed validation which cannot be overridden; this results in an {@link IllegalArgumentException}. */ + default void illegal(String message) { illegal(message, null); } + /** Report a failed validation which cannot be overridden; this results in an {@link IllegalArgumentException}. */ + void illegal(String message, Throwable cause); + /** Report a failed validation which can be overridden; this results in a {@link ValidationException}. */ + void invalid(ValidationId id, String message); + } + + public interface ChangeContext extends Context { + /** The previous model, if any. */ + VespaModel previousModel(); + /** + * Report an action the user must take to change to the new configuration. + * If the action has a {@link ValidationId}, {@link #invalid} is also called for this id, and the action's message. + */ + void require(ConfigChangeAction action); + } - private static class Execution { + static class Execution implements ChangeContext { + private final List<String> errors = new ArrayList<>(); private final Map<ValidationId, List<String>> failures = new LinkedHashMap<>(); private final VespaModel model; private final DeployState deployState; private final List<ConfigChangeAction> actions = new ArrayList<>(); - private Execution(VespaModel model, DeployState deployState) { + Execution(VespaModel model, DeployState deployState) { this.model = model; this.deployState = deployState; } - private void run(Validator validator) { - try { - validator.validate(model, deployState); + void throwIfFailed() { + Optional<ValidationException> invalidException = deployState.validationOverrides().invalidException(failures, deployState.now()); + if (invalidException.isPresent() && deployState.isHosted() && deployState.zone().environment().isManuallyDeployed()) { + deployState.getDeployLogger().logApplicationPackage(Level.WARNING, + "Auto-overriding validation which would be disallowed in production: " + + Exceptions.toMessageString(invalidException.get())); + invalidException = Optional.empty(); } - catch (ValidationException e) { - e.messagesById().forEach((id, messages) -> failures.computeIfAbsent(id, __ -> new ArrayList<>()).addAll(messages)); + + if ( ! errors.isEmpty()) { + String illegalMessage = errors.size() == 1 ? errors.get(0) + : "multiple errors:\n\t" + String.join("\n\t", errors); + if (invalidException.isPresent()) + illegalMessage += "\n" + invalidException.get().getMessage(); + + throw new IllegalArgumentException(illegalMessage); } + + invalidException.ifPresent(e -> { throw e; }); } - private void run(ChangeValidator validator, VespaModel previousModel) { - try { - // Some change validators throw, while some return a list of changes that may again be disallowed. - for (ConfigChangeAction action : validator.validate(previousModel, model, deployState)) { - actions.add(action); - if (action.validationId().isPresent()) run(new Validator() { // Changes without a validation ID are always allowed. - @Override public void validate(VespaModel model, DeployState deployState) { - deployState.validationOverrides().invalid(action.validationId().get(), action.getMessage(), deployState.now()); - } - }); - } - } - catch (ValidationException e) { - e.messagesById().forEach((id, messages) -> failures.computeIfAbsent(id, __ -> new ArrayList<>()).addAll(messages)); - } + List<ConfigChangeAction> actions() { + return actions; } - private void throwIfFailed() { - try { - if (failures.size() == 1 && failures.values().iterator().next().size() == 1) // Retain single-form exception message when possible. - deployState.validationOverrides().invalid(failures.keySet().iterator().next(), failures.values().iterator().next().get(0), deployState.now()); - else - deployState.validationOverrides().invalid(failures, deployState.now()); - } - catch (ValidationException e) { - if (deployState.isHosted() && deployState.zone().environment().isManuallyDeployed()) - deployState.getDeployLogger().logApplicationPackage(Level.WARNING, - "Auto-overriding validation which would be disallowed in production: " + - Exceptions.toMessageString(e)); - else throw e; - } + List<String> errors() { + return errors; + } + + @Override + public DeployState deployState() { + return deployState; + } + + @Override + public VespaModel model() { + return model; + } + + @Override + public VespaModel previousModel() { + return (VespaModel) deployState.getPreviousModel().get(); + } + + @Override + public void require(ConfigChangeAction action) { + actions.add(action); + action.validationId().ifPresent(id -> invalid(id, action.getMessage())); + } + + @Override + public void illegal(String message, Throwable cause) { + if (cause != null) message += ": " + Exceptions.toMessageString(cause); + errors.add(message); + } + + @Override + public void invalid(ValidationId id, String message) { + failures.computeIfAbsent(id, __ -> new ArrayList<>()).add(message); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java index 38d94e55b8e..c56208e5e35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Context; import java.io.Reader; import java.util.Optional; @@ -14,15 +13,15 @@ import java.util.Optional; * * @author hmusum */ -public class ValidationOverridesValidator extends Validator { +public class ValidationOverridesValidator implements Validator { @Override - public void validate(VespaModel model, DeployState deployState) { - Optional<Reader> overrides = deployState.getApplicationPackage().getValidationOverrides(); + public void validate(Context context) { + Optional<Reader> overrides = context.deployState().getApplicationPackage().getValidationOverrides(); if (overrides.isEmpty()) return; ValidationOverrides validationOverrides = ValidationOverrides.fromXml(overrides.get()); - validationOverrides.validate(deployState.now()); + validationOverrides.validate(context.deployState().now(), context::illegal); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java index c678938b5d9..fb1bf0b0ed8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java @@ -1,22 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; - /** * Abstract superclass of all application package validators. * * @author hmusum */ -public abstract class Validator { +public interface Validator { /** - * Validates the input vespamodel - * - * @param model a VespaModel object - * @param deployState the {@link DeployState} built from building the model + * Validates the input Vespa model; illegal configuration should be reported through the context, + * while other problems (system error, insufficient quota, etc.) should be thrown. */ - public abstract void validate(VespaModel model, DeployState deployState); + void validate(Validation.Context context); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java index c9e1a3bdea7..97e422c1a6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java @@ -2,14 +2,13 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.http.Client; import java.security.cert.X509Certificate; import java.util.Collection; import java.util.List; +import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -24,22 +23,20 @@ public class CertificateRemovalChangeValidator implements ChangeValidator { private static final Logger logger = Logger.getLogger(CertificateRemovalChangeValidator.class.getName()); @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { + public void validate(ChangeContext context) { // Skip for tester applications - if (current.applicationPackage().getApplicationId().instance().isTester()) return List.of(); - current.getContainerClusters() + if (context.previousModel().applicationPackage().getApplicationId().instance().isTester()) return; + context.previousModel().getContainerClusters() .forEach((clusterId, currentCluster) -> { - if(next.getContainerClusters().containsKey(clusterId)) + if(context.model().getContainerClusters().containsKey(clusterId)) validateClients(clusterId, currentCluster.getClients(), - next.getContainerClusters().get(clusterId).getClients(), - deployState); + context.model().getContainerClusters().get(clusterId).getClients(), + context::invalid); }); - - return List.of(); } - void validateClients(String clusterId, List<Client> current, List<Client> next, DeployState deployState) { + void validateClients(String clusterId, List<Client> current, List<Client> next, BiConsumer<ValidationId, String> reporter) { List<X509Certificate> currentCertificates = current.stream() .filter(client -> !client.internal()) .map(Client::certificates) @@ -58,12 +55,11 @@ public class CertificateRemovalChangeValidator implements ChangeValidator { List<X509Certificate> missingCerts = currentCertificates.stream().filter(cert -> !nextCertificates.contains(cert)).toList(); if (!missingCerts.isEmpty()) { - deployState.validationOverrides().invalid(ValidationId.certificateRemoval, - "Data plane certificate(s) from cluster '" + clusterId + "' is removed " + - "(removed certificates: " + missingCerts.stream().map(x509Certificate -> x509Certificate.getSubjectX500Principal().getName()).toList() + ") " + - "This can cause client connection issues.", - deployState.now()); + reporter.accept(ValidationId.certificateRemoval, + "Data plane certificate(s) from cluster '" + clusterId + "' is removed " + + "(removed certificates: " + missingCerts.stream().map(x509Certificate -> x509Certificate.getSubjectX500Principal().getName()).toList() + ") " + + "This can cause client connection issues."); } - } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java index 107128fdd89..03479db88d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; +import com.yahoo.vespa.model.application.validation.Validator; -import java.time.Instant; import java.util.List; /** @@ -17,15 +15,9 @@ import java.util.List; public interface ChangeValidator { /** - * Validates the current active vespa model with the next model. - * Both current and next should be non-null. - * - * @param current the current active model - * @param next the next model we would like to activate - * @return a list of actions specifying what needs to be done in order to activate the new model. - * Return an empty list if nothing needs to be done - * @throws IllegalArgumentException if the change fails validation + * Validates changes from the previous to the next model. Necessary actions by the user + * should be reported through the context; see {@link Validator} for more details. */ - List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState); + void validate(ChangeContext context); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index 73d6b9509cb..bfd100f40c9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -5,17 +5,17 @@ import com.yahoo.config.ChangesRequiringRestart; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.Service; -import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Stream; @@ -32,13 +32,13 @@ public class ConfigValueChangeValidator implements ChangeValidator { /** Inspects the configuration in the new and old Vespa model to determine which services that require restart */ @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - return findConfigChangesFromModels(currentModel, nextModel, deployState.getDeployLogger()).toList(); + public void validate(ChangeContext context) { + findConfigChangesFromModels(context.previousModel(), context.model(), context.deployState().getDeployLogger()).forEach(context::require); } - public Stream<ConfigChangeAction> findConfigChangesFromModels(AbstractConfigProducerRoot currentModel, - AbstractConfigProducerRoot nextModel, - DeployLogger logger) { + Stream<ConfigChangeAction> findConfigChangesFromModels(AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel, + DeployLogger logger) { return nextModel.getDescendantServices().stream() .map(service -> findConfigChangeActionForService(service, currentModel, nextModel, logger)) .filter(Optional::isPresent) @@ -136,8 +136,8 @@ public class ConfigValueChangeValidator implements ChangeValidator { return Optional.of(ReflectionUtil.getChangesRequiringRestart(currentConfig.get(), nextConfig.get())); } - private static boolean hasConfigFieldsFlaggedWithRestart( - Class<? extends ConfigInstance> configClass, Class<? extends Service> serviceClass) { + private static boolean hasConfigFieldsFlaggedWithRestart(Class<? extends ConfigInstance> configClass, + Class<? extends Service> serviceClass) { if (!ReflectionUtil.hasRestartMethods(configClass)) { throw new IllegalStateException(String.format( "%s is listed as restart config for %s but does not contain any restart inspection methods.", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java index 2a4e8a2a2a6..58b849d90e6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -2,19 +2,16 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; -import java.util.ArrayList; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static java.util.stream.Collectors.toUnmodifiableSet; @@ -26,18 +23,16 @@ import static java.util.stream.Collectors.toUnmodifiableSet; public class ContainerRestartValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - boolean nodesUnchanged = currentModel.allocatedHosts().equals(nextModel.allocatedHosts()); - boolean contentUnchanged = contentHostsOf(currentModel).equals(contentHostsOf(nextModel)); - List<ConfigChangeAction> actions = new ArrayList<>(); - for (ContainerCluster<ApplicationContainer> cluster : nextModel.getContainerClusters().values()) { - actions.addAll(cluster.getContainers().stream() - .filter(container -> isExistingContainer(container, currentModel)) - .filter(container -> shouldContainerRestartOnDeploy(container, nextModel)) - .map(container -> createConfigChangeAction(cluster.id(), container, nextModel, nodesUnchanged, contentUnchanged)) - .toList()); + public void validate(ChangeContext context) { + boolean nodesUnchanged = context.previousModel().allocatedHosts().equals(context.model().allocatedHosts()); + boolean contentUnchanged = contentHostsOf(context.previousModel()).equals(contentHostsOf(context.model())); + for (ContainerCluster<ApplicationContainer> cluster : context.model().getContainerClusters().values()) { + cluster.getContainers().stream() + .filter(container -> isExistingContainer(container, context.previousModel())) + .filter(container -> shouldContainerRestartOnDeploy(container, context.model())) + .map(container -> createConfigChangeAction(cluster.id(), container, context.model(), nodesUnchanged, contentUnchanged)) + .forEach(context::require); } - return actions; } private Set<HostSpec> contentHostsOf(VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java index fb48ec68c12..3a0aa0afd76 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -1,18 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.ArrayList; -import java.util.List; - /** * Checks that this does not remove a content cluster (or changes its id) * as that means losing all data of that cluster. @@ -22,24 +16,22 @@ import java.util.List; public class ContentClusterRemovalValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - List<ConfigChangeAction> actions = new ArrayList<>(); - for (String currentClusterId : current.getContentClusters().keySet()) { - ContentCluster nextCluster = next.getContentClusters().get(currentClusterId); + public void validate(ChangeContext context) { + for (String currentClusterId : context.previousModel().getContentClusters().keySet()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentClusterId); if (nextCluster == null) { - deployState.validationOverrides().invalid(ValidationId.contentClusterRemoval, - "Content cluster '" + currentClusterId + "' is removed. " + - "This will cause loss of all data in this cluster", - deployState.now()); + context.invalid(ValidationId.contentClusterRemoval, + "Content cluster '" + currentClusterId + "' is removed. " + + "This will cause loss of all data in this cluster"); // If we allow the removal, we must restart all containers to ensure mbus is OK. - for (ApplicationContainerCluster cluster : next.getContainerClusters().values()) { - actions.add(new VespaRestartAction(cluster.id(), - "Content cluster '" + currentClusterId + "' has been removed", - cluster.getContainers().stream().map(ApplicationContainer::getServiceInfo).toList())); + for (ApplicationContainerCluster cluster : context.model().getContainerClusters().values()) { + context.require(new VespaRestartAction(cluster.id(), + "Content cluster '" + currentClusterId + "' has been removed", + cluster.getContainers().stream().map(ApplicationContainer::getServiceInfo).toList())); } } } - return actions; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java index fec08f90b1e..34e5a4b42a5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java @@ -2,14 +2,10 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.List; - /** * Checks that this does not remove a data type in a cluster, as that causes deletion * of all data of that type. @@ -19,22 +15,20 @@ import java.util.List; public class ContentTypeRemovalValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - for (ContentCluster currentCluster : current.getContentClusters().values()) { - ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getSubId()); + public void validate(ChangeContext context) { + for (ContentCluster currentCluster : context.previousModel().getContentClusters().values()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentCluster.getSubId()); if (nextCluster == null) continue; // validated elsewhere for (NewDocumentType type : currentCluster.getDocumentDefinitions().values()) { if ( ! nextCluster.getDocumentDefinitions().containsKey(type.getName())) { - deployState.validationOverrides().invalid(ValidationId.contentTypeRemoval, - "Schema '" + type.getName() + "' is removed " + - "in content cluster '" + currentCluster.getName() + "'. " + - "This will cause loss of all data in this schema", - deployState.now()); + context.invalid(ValidationId.contentTypeRemoval, + "Schema '" + type.getName() + "' is removed " + + "in content cluster '" + currentCluster.getName() + "'. " + + "This will cause loss of all data in this schema"); } } } - return List.of(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java index 0590bb2d1e6..df8bf0e9b01 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java @@ -2,35 +2,30 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.List; import java.util.Map; /** - * Class that fails via exception if global attribute changes for a document + * Class that adds a validation failure if global attribute changes for a document * type in a content cluster unless corresponding override is present. */ public class GlobalDocumentChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - if (!deployState.validationOverrides().allows(ValidationId.globalDocumentChange.value(), deployState.now())) { - for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) { - ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey()); + public void validate(ChangeContext context) { + for (Map.Entry<String, ContentCluster> currentEntry : context.previousModel().getContentClusters().entrySet()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentEntry.getKey()); if (nextCluster == null) continue; - validateContentCluster(currentEntry.getValue(), nextCluster); + validateContentCluster(context, currentEntry.getValue(), nextCluster); } - } - return List.of(); } - private void validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster) { + private void validateContentCluster(ChangeContext context, ContentCluster currentCluster, ContentCluster nextCluster) { String clusterName = currentCluster.getName(); currentCluster.getDocumentDefinitions().forEach((documentTypeName, currentDocumentType) -> { NewDocumentType nextDocumentType = nextCluster.getDocumentDefinitions().get(documentTypeName); @@ -38,10 +33,12 @@ public class GlobalDocumentChangeValidator implements ChangeValidator { boolean currentIsGlobal = currentCluster.isGloballyDistributed(currentDocumentType); boolean nextIsGlobal = nextCluster.isGloballyDistributed(nextDocumentType); if (currentIsGlobal != nextIsGlobal) { - throw new IllegalStateException(String.format("Document type %s in cluster %s changed global from %s to %s. " + - "Add validation override '%s' to force this change through. " + - "First, stop services on all content nodes. Then, deploy with validation override. Finally, start services on all content nodes.", - documentTypeName, clusterName, currentIsGlobal, nextIsGlobal, ValidationId.globalDocumentChange.value())); + String reason = "Document type %s in cluster %s changed global from %s to %s. ".formatted(documentTypeName, clusterName, currentIsGlobal, nextIsGlobal) + + "To handle this change, first stop services on all content nodes. Then, deploy with validation override. Finally, start services on all content nodes"; + if ( ! context.deployState().validationOverrides().allows(ValidationId.globalDocumentChange, context.deployState().now())) + context.invalid(ValidationId.globalDocumentChange, reason); + else if (context.deployState().isHosted()) + context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason)); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index cc28be928ec..f4477bdb141 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -5,15 +5,14 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.application.validation.change.search.DocumentDatabaseChangeValidator; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,15 +27,13 @@ import java.util.stream.Collectors; public class IndexedSearchClusterChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - List<ConfigChangeAction> result = new ArrayList<>(); - for (Map.Entry<String, ContentCluster> currentEntry : current.getContentClusters().entrySet()) { - ContentCluster nextCluster = next.getContentClusters().get(currentEntry.getKey()); + public void validate(ChangeContext context) { + for (Map.Entry<String, ContentCluster> currentEntry : context.previousModel().getContentClusters().entrySet()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentEntry.getKey()); if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) { - result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, deployState)); + validateContentCluster(currentEntry.getValue(), nextCluster, context.deployState()).forEach(context::require); } } - return result; } private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, @@ -88,18 +85,16 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { } private static List<ServiceInfo> getSearchNodeServices(IndexedSearchCluster cluster) { - return cluster.getSearchNodes().stream(). - map(node -> node.getServiceInfo()). - toList(); + return cluster.getSearchNodes().stream().map(AbstractService::getServiceInfo).toList(); } private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, List<ServiceInfo> services, String docTypeName) { - return result.stream(). - map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(), - services, docTypeName)). - collect(Collectors.toList()); + return result.stream() + .map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(), + services, docTypeName)) + .collect(Collectors.toList()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index 9621619f888..91ada2b602f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -4,9 +4,8 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.SearchNode; @@ -16,7 +15,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toCollection; /** * Returns any change to the indexing mode of a cluster. @@ -27,14 +27,12 @@ import java.util.stream.Collectors; public class IndexingModeChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - List<ConfigChangeAction> actions = new ArrayList<>(); - for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) { - ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey()); + public void validate(ChangeContext context) { + for (Map.Entry<String, ContentCluster> currentEntry : context.previousModel().getContentClusters().entrySet()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentEntry.getKey()); if (nextCluster == null) continue; - actions.addAll(validateContentCluster(currentEntry.getValue(), nextCluster)); + validateContentCluster(currentEntry.getValue(), nextCluster).forEach(context::require); } - return actions; } private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster) { @@ -88,7 +86,7 @@ public class IndexingModeChangeValidator implements ChangeValidator { private static Set<String> toDocumentTypeNames(List<NewDocumentType> types) { return types.stream() .map(type -> type.getFullName().getName()) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + .collect(toCollection(LinkedHashSet::new)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java index 0bb30436272..0d4776ad00a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -2,15 +2,14 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -26,16 +25,14 @@ import java.util.stream.Collectors; public class NodeResourceChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - var restartActions = new ArrayList<ConfigChangeAction>(); - for (ClusterSpec.Id clusterId : current.allClusters()) { - Optional<NodeResources> currentResources = resourcesOf(clusterId, current); - Optional<NodeResources> nextResources = resourcesOf(clusterId, next); + public void validate(ChangeContext context) { + for (ClusterSpec.Id clusterId : context.previousModel().allClusters()) { + Optional<NodeResources> currentResources = resourcesOf(clusterId, context.previousModel()); + Optional<NodeResources> nextResources = resourcesOf(clusterId, context.model()); if (currentResources.isEmpty() || nextResources.isEmpty()) continue; // new or removed cluster if ( changeRequiresRestart(currentResources.get(), nextResources.get())) - restartActions.addAll(createRestartActionsFor(clusterId, current)); + createRestartActionsFor(clusterId, context.previousModel()).forEach(context::require); } - return restartActions; } private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java index 54e64d82921..3fc3eafcc98 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java @@ -2,11 +2,8 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.List; /** * Checks that redundancy is not increased (without a validation override), @@ -17,21 +14,19 @@ import java.util.List; public class RedundancyIncreaseValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - for (ContentCluster currentCluster : current.getContentClusters().values()) { - ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getSubId()); + public void validate(ChangeContext context) { + for (ContentCluster currentCluster : context.previousModel().getContentClusters().values()) { + ContentCluster nextCluster = context.model().getContentClusters().get(currentCluster.getSubId()); if (nextCluster == null) continue; if (redundancyOf(nextCluster) > redundancyOf(currentCluster)) { - deployState.validationOverrides().invalid(ValidationId.redundancyIncrease, - "Increasing redundancy from " + redundancyOf(currentCluster) + " to " + - redundancyOf(nextCluster) + " in '" + currentCluster + ". " + - "This is a safe operation but verify that you have room for a " + - redundancyOf(nextCluster) + "/" + redundancyOf(currentCluster) + "x increase " + - "in content size", - deployState.now()); + context.invalid(ValidationId.redundancyIncrease, + "Increasing redundancy from " + redundancyOf(currentCluster) + " to " + + redundancyOf(nextCluster) + " in '" + currentCluster + ". " + + "This is a safe operation but verify that you have room for a " + + redundancyOf(nextCluster) + "/" + redundancyOf(currentCluster) + "x increase " + + "in content size"); } } - return List.of(); } private int redundancyOf(ContentCluster cluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index a09a730b00c..5d7a8779005 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -2,14 +2,11 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.model.VespaModel; - -import java.util.List; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; /** * Checks that no cluster sizes are reduced too much in one go. @@ -19,31 +16,26 @@ import java.util.List; public class ResourcesReductionValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - for (var clusterId : current.allClusters()) { - if (next.allClusters().contains(clusterId)) - validate(clusterId, current, next, deployState); + public void validate(ChangeContext context) { + for (var clusterId : context.previousModel().allClusters()) { + if (context.model().allClusters().contains(clusterId)) + validate(clusterId, context); } - return List.of(); } - private void validate(ClusterSpec.Id clusterId, - VespaModel currentModel, - VespaModel nextModel, - DeployState deployState) { - ClusterResources current = clusterResources(clusterId, currentModel); - ClusterResources next = clusterResources(clusterId, nextModel); + private void validate(ClusterSpec.Id clusterId, ChangeContext context) { + ClusterResources current = clusterResources(clusterId, context.previousModel()); + ClusterResources next = clusterResources(clusterId, context.model()); if (current == null || next == null) return; // No request recording - test if (current.nodeResources().isUnspecified() || next.nodeResources().isUnspecified()) { // Self-hosted - unspecified resources; compare node count int currentNodes = current.nodes(); int nextNodes = next.nodes(); if (nextNodes < 0.5 * currentNodes && nextNodes != currentNodes - 1) { - deployState.validationOverrides().invalid(ValidationId.resourcesReduction, - "Size reduction in '" + clusterId.value() + "' is too large: " + - "To guard against mistakes, the new max nodes must be at least 50% of the current nodes. " + - "Current nodes: " + currentNodes + ", new nodes: " + nextNodes, - deployState.now()); + context.invalid(ValidationId.resourcesReduction, + "Size reduction in '" + clusterId.value() + "' is too large: " + + "To guard against mistakes, the new max nodes must be at least 50% of the current nodes. " + + "Current nodes: " + currentNodes + ", new nodes: " + nextNodes); } } else { @@ -52,13 +44,12 @@ public class ResourcesReductionValidator implements ChangeValidator { if (nextResources.vcpu() < 0.5 * currentResources.vcpu() || nextResources.memoryGb() < 0.5 * currentResources.memoryGb() || nextResources.diskGb() < 0.5 * currentResources.diskGb()) - deployState.validationOverrides().invalid(ValidationId.resourcesReduction, - "Resource reduction in '" + clusterId.value() + "' is too large: " + - "To guard against mistakes, the new max resources must be at least 50% of the current " + - "max resources in all dimensions. " + - "Current: " + currentResources.withBandwidthGbps(0) + // (don't output bandwidth here) - ", new: " + nextResources.withBandwidthGbps(0), - deployState.now()); + context.invalid(ValidationId.resourcesReduction, + "Resource reduction in '" + clusterId.value() + "' is too large: " + + "To guard against mistakes, the new max resources must be at least 50% of the current " + + "max resources in all dimensions. " + + "Current: " + currentResources.withBandwidthGbps(0) + // (don't output bandwidth here) + ", new: " + nextResources.withBandwidthGbps(0)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index e118a2940d7..373bfe24984 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -4,9 +4,8 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.OnnxModelCost; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import java.util.ArrayList; @@ -16,10 +15,10 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Logger; +import static com.yahoo.config.model.api.OnnxModelCost.ModelInfo; import static com.yahoo.vespa.model.application.validation.JvmHeapSizeValidator.gbLimit; import static com.yahoo.vespa.model.application.validation.JvmHeapSizeValidator.percentLimit; import static java.util.logging.Level.FINE; -import static com.yahoo.config.model.api.OnnxModelCost.ModelInfo; import static java.util.logging.Level.INFO; /** @@ -34,28 +33,26 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida private static final Logger log = Logger.getLogger(RestartOnDeployForOnnxModelChangesValidator.class.getName()); @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - if ( ! deployState.featureFlags().restartOnDeployWhenOnnxModelChanges()) return List.of(); - List<ConfigChangeAction> actions = new ArrayList<>(); + public void validate(ChangeContext context) { + if ( ! context.deployState().featureFlags().restartOnDeployWhenOnnxModelChanges()) return; // Compare onnx models used by each cluster and set restart on deploy for cluster if estimated cost, // model hash or model options have changed - for (var cluster : nextModel.getContainerClusters().values()) { - var clusterInCurrentModel = currentModel.getContainerClusters().get(cluster.getName()); + for (var cluster : context.model().getContainerClusters().values()) { + var clusterInCurrentModel = context.previousModel().getContainerClusters().get(cluster.getName()); if (clusterInCurrentModel == null) continue; var currentModels = clusterInCurrentModel.onnxModelCostCalculator().models(); var nextModels = cluster.onnxModelCostCalculator().models(); - if (enoughMemoryToAvoidRestart(clusterInCurrentModel, cluster, deployState.getDeployLogger())) + if (enoughMemoryToAvoidRestart(clusterInCurrentModel, cluster, context.deployState().getDeployLogger())) continue; log.log(FINE, "Validating %s, current Onnx models:%s, next Onnx models:%s" .formatted(cluster, currentModels, nextModels)); - actions.addAll(validateModelChanges(cluster, currentModels, nextModels)); - actions.addAll(validateSetOfModels(cluster, currentModels, nextModels)); + validateModelChanges(cluster, currentModels, nextModels).forEach(context::require); + validateSetOfModels(cluster, currentModels, nextModels).forEach(context::require); } - return actions; } private List<ConfigChangeAction> validateModelChanges(ApplicationContainerCluster cluster, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java index aac9ef28cdb..0eee74fbe32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.Service; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.stream.Stream; @@ -22,8 +20,8 @@ import java.util.stream.Stream; public class StartupCommandChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, DeployState deployState) { - return findServicesWithChangedStartupCommand(currentModel, nextModel).toList(); + public void validate(ChangeContext context) { + findServicesWithChangedStartupCommand(context.previousModel(), context.model()).forEach(context::require); } public Stream<ConfigChangeAction> findServicesWithChangedStartupCommand(AbstractConfigProducerRoot currentModel, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 10848947ee1..3b89467299d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.schema.derived.AttributeFields; import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -28,20 +27,18 @@ import java.util.stream.Collectors; public class StreamingSearchClusterChangeValidator implements ChangeValidator { @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - List<ConfigChangeAction> result = new ArrayList<>(); - current.getContentClusters().forEach((clusterName, currentCluster) -> { - ContentCluster nextCluster = next.getContentClusters().get(clusterName); + public void validate(ChangeContext context) { + context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { + ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters(); currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> { Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters); - nextStreamingCluster.ifPresent(streamingSearchCluster -> result.addAll(validateStreamingCluster(currentCluster, currentStreamingCluster, - nextCluster, streamingSearchCluster))); + nextStreamingCluster.ifPresent(streamingSearchCluster -> validateStreamingCluster(currentCluster, currentStreamingCluster, + nextCluster, streamingSearchCluster).forEach(context::require)); }); } }); - return result; } private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java index d088c9e67ff..cf552d05e28 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyValidator.java @@ -2,14 +2,14 @@ package com.yahoo.vespa.model.application.validation.first; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; +import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.application.validation.Validator; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.List; import java.util.stream.Stream; /** @@ -17,24 +17,23 @@ import java.util.stream.Stream; * * @author bratseth */ -public class RedundancyValidator extends Validator implements ChangeValidator { +public class RedundancyValidator implements Validator, ChangeValidator { /** Validate on first deployment. */ @Override - public void validate(VespaModel model, DeployState deployState) { - if ( ! shouldValidate(deployState)) return; - clustersWithRedundancyOne(model).forEach(cluster -> invalidRedundancy(cluster, deployState)); + public void validate(Context context) { + if ( ! shouldValidate(context.deployState())) return; + clustersWithRedundancyOne(context.model()).forEach(cluster -> invalidRedundancy(cluster, context)); } /** Validate on change. */ @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - if ( ! shouldValidate(deployState)) return List.of(); + public void validate(ChangeContext context) { + if ( ! shouldValidate(context.deployState())) return; - clustersWithRedundancyOne(next) - .filter(cluster -> ! hasRedundancyOne(current.getContentClusters().get(cluster.id().value()))) - .forEach(cluster -> invalidRedundancy(cluster, deployState)); - return List.of(); + clustersWithRedundancyOne(context.model()) + .filter(cluster -> ! hasRedundancyOne(context.previousModel().getContentClusters().get(cluster.id().value()))) + .forEach(cluster -> invalidRedundancy(cluster, context)); } private boolean shouldValidate(DeployState deployState) { @@ -49,12 +48,11 @@ public class RedundancyValidator extends Validator implements ChangeValidator { return cluster != null && cluster.getRedundancy().finalRedundancy() == 1 && cluster.getRedundancy().groups() == 1; } - private void invalidRedundancy(ContentCluster cluster, DeployState deployState) { - deployState.validationOverrides().invalid(ValidationId.redundancyOne, - cluster + " has redundancy 1, which will cause it to lose data " + - "if a node fails. This requires an override on first deployment " + - "in a production zone", - deployState.now()); + private void invalidRedundancy(ContentCluster cluster, Context context) { + context.invalid(ValidationId.redundancyOne, + cluster + " has redundancy 1, which will cause it to lose data " + + "if a node fails. This requires an override on first deployment " + + "in a production zone"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index 72b72c369dc..af7c7478745 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -10,6 +10,7 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BertEmbedder; import com.yahoo.vespa.model.container.component.ColBertEmbedder; +import com.yahoo.vespa.model.container.component.SpladeEmbedder; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; @@ -50,6 +51,7 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde case "hugging-face-tokenizer" -> new HuggingFaceTokenizer(spec, state); case "colbert-embedder" -> new ColBertEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "bert-embedder" -> new BertEmbedder((ApplicationContainerCluster)ancestor, spec, state); + case "splade-embedder" -> new SpladeEmbedder((ApplicationContainerCluster)ancestor, spec, state); default -> throw new IllegalArgumentException("Unknown component type '%s'".formatted(type)); }; } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 8c4adfb96cb..9896ca95e97 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -16,6 +16,7 @@ import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.TenantName; import com.yahoo.container.bundle.BundleInstantiationSpecification; @@ -139,7 +140,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat : defaultHeapSizePercentageOfAvailableMemory; onnxModelCost = deployState.onnxModelCost(); onnxModelCostCalculator = deployState.onnxModelCost().newCalculator( - deployState.getApplicationPackage(), deployState.getProperties().applicationId()); + deployState.getApplicationPackage(), deployState.getProperties().applicationId(), ClusterSpec.Id.from(clusterId)); logger = deployState.getDeployLogger(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java index 67fb720b8c0..f546f5060ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java @@ -8,10 +8,14 @@ import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import java.util.Set; + import static com.yahoo.embedding.BertBaseEmbedderConfig.OnnxExecutionMode; import static com.yahoo.embedding.BertBaseEmbedderConfig.PoolingStrategy; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.BERT_VOCAB; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.ONNX_MODEL; /** * @author bjorncs @@ -32,14 +36,14 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf public BertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.BertBaseEmbedder", INTEGRATION_BUNDLE_NAME, xml); - var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + var model = Model.fromXml(state, xml, "transformer-model", Set.of(ONNX_MODEL)).orElseThrow(); this.onnxModelOptions = new OnnxModelOptions( getChildValue(xml, "onnx-execution-mode"), getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); - vocabRef = Model.fromXml(state, xml, "tokenizer-vocab").orElseThrow().modelReference(); + vocabRef = Model.fromXml(state, xml, "tokenizer-vocab", Set.of(BERT_VOCAB)).orElseThrow().modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index d22e6afc3d1..abca3290a31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -8,9 +8,13 @@ import com.yahoo.embedding.ColBertEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import java.util.Set; + import static com.yahoo.embedding.ColBertEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.HF_TOKENIZER; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.ONNX_MODEL; /** @@ -37,16 +41,14 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); - var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + var model = Model.fromXml(state, xml, "transformer-model", Set.of(ONNX_MODEL)).orElseThrow(); this.onnxModelOptions = new OnnxModelOptions( getChildValue(xml, "onnx-execution-mode"), getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); - vocabRef = Model.fromXml(state, xml, "tokenizer-model") - .map(Model::modelReference) - .orElseGet(() -> resolveDefaultVocab(model, state)); + vocabRef = Model.fromXmlOrImplicitlyFromOnnxModel(state, xml, model, "tokenizer-model", Set.of(HF_TOKENIZER)).modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); maxQueryTokens = getChildValue(xml, "max-query-tokens").map(Integer::parseInt).orElse(null); maxDocumentTokens = getChildValue(xml, "max-document-tokens").map(Integer::parseInt).orElse(null); @@ -59,14 +61,6 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo model.registerOnnxModelCost(cluster, onnxModelOptions); } - private static ModelReference resolveDefaultVocab(Model model, DeployState state) { - var modelId = model.modelId().orElse(null); - if (state.isHosted() && modelId != null) { - return Model.fromParams(state, model.name(), modelId + "-vocab", null, null).modelReference(); - } - throw new IllegalArgumentException("'tokenizer-model' must be specified"); - } - @Override public void getConfig(ColBertEmbedderConfig.Builder b) { b.transformerModel(modelRef).tokenizerPath(vocabRef); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java index d98c72ab3a4..fe0bb7c8075 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java @@ -8,10 +8,14 @@ import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import java.util.Set; + import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.PoolingStrategy; import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.HF_TOKENIZER; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.ONNX_MODEL; /** @@ -32,16 +36,14 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm public HuggingFaceEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.huggingface.HuggingFaceEmbedder", INTEGRATION_BUNDLE_NAME, xml); - var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + var model = Model.fromXml(state, xml, "transformer-model", Set.of(ONNX_MODEL)).orElseThrow(); this.onnxModelOptions = new OnnxModelOptions( getChildValue(xml, "onnx-execution-mode"), getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); - vocabRef = Model.fromXml(state, xml, "tokenizer-model") - .map(Model::modelReference) - .orElseGet(() -> resolveDefaultVocab(model, state)); + vocabRef = Model.fromXmlOrImplicitlyFromOnnxModel(state, xml, model, "tokenizer-model", Set.of(HF_TOKENIZER)).modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); @@ -52,14 +54,6 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm model.registerOnnxModelCost(cluster, onnxModelOptions); } - private static ModelReference resolveDefaultVocab(Model model, DeployState state) { - var modelId = model.modelId().orElse(null); - if (state.isHosted() && modelId != null) { - return Model.fromParams(state, model.name(), modelId + "-vocab", null, null).modelReference(); - } - throw new IllegalArgumentException("'tokenizer-model' must be specified"); - } - @Override public void getConfig(HuggingFaceEmbedderConfig.Builder b) { b.transformerModel(modelRef).tokenizerPath(vocabRef); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java index f808916d83b..c73cf9dce37 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java @@ -10,9 +10,11 @@ import com.yahoo.text.XML; import org.w3c.dom.Element; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.LINGUISTICS_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.HF_TOKENIZER; /** * @author bjorncs @@ -25,7 +27,7 @@ public class HuggingFaceTokenizer extends TypedComponent implements HuggingFaceT super("com.yahoo.language.huggingface.HuggingFaceTokenizer", LINGUISTICS_BUNDLE_NAME, xml); for (Element element : XML.getChildren(xml, "model")) { var lang = element.hasAttribute("language") ? element.getAttribute("language") : "unknown"; - langToModel.put(lang, Model.fromXml(state, element).modelReference()); + langToModel.put(lang, Model.fromXml(state, element, Set.of(HF_TOKENIZER)).modelReference()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java index 102ed926fad..7d6285d00c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java @@ -15,6 +15,7 @@ import org.w3c.dom.Element; import java.net.URI; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * Represents a model, e.g ONNX model for an embedder. @@ -28,7 +29,7 @@ class Model { private final ApplicationFile file; private final ModelReference ref; - private Model(DeployState ds, String paramName, String modelId, URI url, Path file) { + private Model(DeployState ds, String paramName, String modelId, URI url, Path file, Set<String> requiredTags) { this.paramName = Objects.requireNonNull(paramName); if (modelId == null && url == null && file == null) throw new IllegalArgumentException("At least one of 'model-id', 'url' or 'path' must be specified"); @@ -37,22 +38,35 @@ class Model { this.file = file != null ? ds.getApplicationPackage().getFile(file) : null; this.ref = ModelIdResolver.resolveToModelReference( paramName, Optional.ofNullable(modelId), Optional.ofNullable(url).map(URI::toString), - Optional.ofNullable(file).map(Path::toString), ds); + Optional.ofNullable(file).map(Path::toString), requiredTags, ds); } - static Model fromParams(DeployState ds, String paramName, String modelId, URI url, Path file) { - return new Model(ds, paramName, modelId, url, file); + static Model fromParams(DeployState ds, String paramName, String modelId, URI url, Path file, Set<String> requiredTags) { + return new Model(ds, paramName, modelId, url, file, requiredTags); } - static Optional<Model> fromXml(DeployState ds, Element parent, String name) { - return XmlHelper.getOptionalChild(parent, name).map(e -> fromXml(ds, e)); + static Optional<Model> fromXml(DeployState ds, Element parent, String name, Set<String> requiredTags) { + return XmlHelper.getOptionalChild(parent, name).map(e -> fromXml(ds, e, requiredTags)); } - static Model fromXml(DeployState ds, Element model) { + static Model fromXml(DeployState ds, Element model, Set<String> requiredTags) { var modelId = XmlHelper.getOptionalAttribute(model, "model-id").orElse(null); var url = XmlHelper.getOptionalAttribute(model, "url").map(URI::create).orElse(null); var path = XmlHelper.getOptionalAttribute(model, "path").map(Path::fromString).orElse(null); - return new Model(ds, model.getTagName(), modelId, url, path); + return new Model(ds, model.getTagName(), modelId, url, path, requiredTags); + } + + /** Return tokenizer model from XML if specified, alternatively use model id for ONNX model with suffix '-vocab' appended */ + static Model fromXmlOrImplicitlyFromOnnxModel( + DeployState ds, Element parent, Model onnxModel, String paramName, Set<String> requiredTags) { + return fromXml(ds, parent, paramName, requiredTags) + .orElseGet(() -> { + var modelId = onnxModel.modelId().orElse(null); + if (ds.isHosted() && modelId != null) { + return fromParams(ds, onnxModel.name(), modelId + "-vocab", null, null, requiredTags); + } + throw new IllegalArgumentException("'%s' must be specified".formatted(paramName)); + }); } void registerOnnxModelCost(ApplicationContainerCluster c, OnnxModelOptions onnxModelOptions) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SpladeEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SpladeEmbedder.java new file mode 100644 index 00000000000..9e0a3a0ba5c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SpladeEmbedder.java @@ -0,0 +1,65 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.container.component; +import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.OnnxModelOptions; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.embedding.SpladeEmbedderConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import org.w3c.dom.Element; + +import java.util.Set; + +import static com.yahoo.text.XML.getChildValue; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.HF_TOKENIZER; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.ONNX_MODEL; + +public class SpladeEmbedder extends TypedComponent implements SpladeEmbedderConfig.Producer { + + private final OnnxModelOptions onnxModelOptions; + private final ModelReference modelRef; + private final ModelReference vocabRef; + private final Integer maxTokens; + private final String transformerInputIds; + private final String transformerAttentionMask; + private final String transformerTokenTypeIds; + private final String transformerOutput; + private final Double termScoreThreshold; + + public SpladeEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { + super("ai.vespa.embedding.SpladeEmbedder", INTEGRATION_BUNDLE_NAME, xml); + var model = Model.fromXml(state, xml, "transformer-model", Set.of(ONNX_MODEL)).orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); + modelRef = model.modelReference(); + vocabRef = Model.fromXmlOrImplicitlyFromOnnxModel(state, xml, model, "tokenizer-model", Set.of(HF_TOKENIZER)).modelReference(); + maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); + transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); + transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); + transformerTokenTypeIds = getChildValue(xml, "transformer-token-type-ids").orElse(null); + transformerOutput = getChildValue(xml, "transformer-output").orElse(null); + termScoreThreshold = getChildValue(xml, "term-score-threshold").map(Double::parseDouble).orElse(null); + model.registerOnnxModelCost(cluster, onnxModelOptions); + } + + @Override + public void getConfig(SpladeEmbedderConfig.Builder b) { + b.transformerModel(modelRef).tokenizerPath(vocabRef); + if (maxTokens != null) b.transformerMaxTokens(maxTokens); + if (transformerInputIds != null) b.transformerInputIds(transformerInputIds); + if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); + if (transformerTokenTypeIds != null) b.transformerTokenTypeIds(transformerTokenTypeIds); + if (transformerOutput != null) b.transformerOutput(transformerOutput); + if (termScoreThreshold != null) b.termScoreThreshold(termScoreThreshold); + + onnxModelOptions.executionMode().ifPresent(value -> b.transformerExecutionMode(SpladeEmbedderConfig.TransformerExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::transformerInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::transformerIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.transformerGpuDevice(value.deviceNumber())); + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index 0142b7f246a..9ff9344edcb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -7,10 +7,11 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import org.w3c.dom.Element; -import java.util.Collections; +import java.net.URI; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -21,44 +22,55 @@ import java.util.stream.Collectors; */ public class ModelIdResolver { - private static Map<String, String> setupProvidedModels() { - Map<String, String> models = new HashMap<>(); - models.put("minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx"); - models.put("mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx"); - models.put("bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"); - models.put("flan-t5-vocab", "https://data.vespa.oath.cloud/onnx_models/flan-t5-spiece.model"); - models.put("flan-t5-small-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-encoder-model.onnx"); - models.put("flan-t5-small-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-decoder-model.onnx"); - models.put("flan-t5-base-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-encoder-model.onnx"); - models.put("flan-t5-base-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-decoder-model.onnx"); - models.put("flan-t5-large-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-encoder-model.onnx"); - models.put("flan-t5-large-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-decoder-model.onnx"); + public static final String HF_TOKENIZER = "huggingface-tokenizer"; + public static final String ONNX_MODEL = "onnx-model"; + public static final String BERT_VOCAB = "bert-vocabulary"; - models.put("multilingual-e5-base", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/model.onnx"); - models.put("multilingual-e5-base-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/tokenizer.json"); + private static Map<String, ProvidedModel> setupProvidedModels() { + var m = new HashMap<String, ProvidedModel>(); + register(m, "minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx", Set.of(ONNX_MODEL)); + register(m, "mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx", Set.of(ONNX_MODEL)); + register(m, "bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt", Set.of(BERT_VOCAB)); + register(m, "flan-t5-vocab", "https://data.vespa.oath.cloud/onnx_models/flan-t5-spiece.model", Set.of()); + register(m, "flan-t5-small-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-encoder-model.onnx", Set.of(ONNX_MODEL)); + register(m, "flan-t5-small-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-decoder-model.onnx", Set.of(ONNX_MODEL)); + register(m, "flan-t5-base-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-encoder-model.onnx", Set.of(ONNX_MODEL)); + register(m, "flan-t5-base-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-decoder-model.onnx", Set.of(ONNX_MODEL)); + register(m, "flan-t5-large-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-encoder-model.onnx", Set.of(ONNX_MODEL)); + register(m, "flan-t5-large-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-decoder-model.onnx", Set.of(ONNX_MODEL)); - models.put("multilingual-e5-small", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/model.onnx"); - models.put("multilingual-e5-small-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/tokenizer.json"); + register(m, "multilingual-e5-base", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/model.onnx", Set.of(ONNX_MODEL)); + register(m, "multilingual-e5-base-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-base/tokenizer.json", Set.of(HF_TOKENIZER)); - models.put("multilingual-e5-small-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/model.onnx"); - models.put("multilingual-e5-small-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/tokenizer.json"); + register(m, "multilingual-e5-small", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/model.onnx", Set.of(ONNX_MODEL)); + register(m, "multilingual-e5-small-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/tokenizer.json", Set.of(HF_TOKENIZER)); - models.put("e5-small-v2", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/model.onnx"); - models.put("e5-small-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/tokenizer.json"); + register(m, "multilingual-e5-small-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/model.onnx", Set.of(ONNX_MODEL)); + register(m, "multilingual-e5-small-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/tokenizer.json", Set.of(HF_TOKENIZER)); - models.put("e5-small-v2-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/model.onnx"); - models.put("e5-small-v2-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/tokenizer.json"); + register(m, "e5-small-v2", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/model.onnx", Set.of(ONNX_MODEL)); + register(m, "e5-small-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/tokenizer.json", Set.of(HF_TOKENIZER)); - models.put("e5-base-v2", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/model.onnx"); - models.put("e5-base-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/tokenizer.json"); + register(m, "e5-small-v2-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/model.onnx", Set.of(ONNX_MODEL)); + register(m, "e5-small-v2-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/tokenizer.json", Set.of(HF_TOKENIZER)); - models.put("e5-large-v2", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/model.onnx"); - models.put("e5-large-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/tokenizer.json"); + register(m, "e5-base-v2", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/model.onnx", Set.of(ONNX_MODEL)); + register(m, "e5-base-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/tokenizer.json", Set.of(HF_TOKENIZER)); - return Collections.unmodifiableMap(models); + register(m, "e5-large-v2", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/model.onnx", Set.of(ONNX_MODEL)); + register(m, "e5-large-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/tokenizer.json", Set.of(HF_TOKENIZER)); + return Map.copyOf(m); } - private static final Map<String, String> providedModels = setupProvidedModels(); + private record ProvidedModel(String modelId, URI uri, Set<String> tags) { + ProvidedModel { tags = Set.copyOf(tags); } + } + + private static void register(Map<String, ProvidedModel> models, String modelId, String uri, Set<String> tags) { + models.put(modelId, new ProvidedModel(modelId, URI.create(uri), tags)); + } + + private static final Map<String, ProvidedModel> providedModels = setupProvidedModels(); /** * Finds any config values of type 'model' below the given config element and @@ -79,7 +91,7 @@ public class ModelIdResolver { if ( ! value.hasAttribute("model-id")) return; if (hosted) { - value.setAttribute("url", modelIdToUrl(value.getTagName(), value.getAttribute("model-id"))); + value.setAttribute("url", modelIdToUrl(value.getTagName(), value.getAttribute("model-id"), Set.of())); value.removeAttribute("path"); } else if ( ! value.hasAttribute("url") && ! value.hasAttribute("path")) { @@ -88,10 +100,10 @@ public class ModelIdResolver { } public static ModelReference resolveToModelReference( - String paramName, Optional<String> id, Optional<String> url, Optional<String> path, DeployState state) { + String paramName, Optional<String> id, Optional<String> url, Optional<String> path, Set<String> requiredTags, DeployState state) { if (id.isEmpty()) return createModelReference(Optional.empty(), url, path, state); else if (state.isHosted()) - return createModelReference(id, Optional.of(modelIdToUrl(paramName, id.get())), Optional.empty(), state); + return createModelReference(id, Optional.of(modelIdToUrl(paramName, id.get(), requiredTags)), Optional.empty(), state); else if (url.isEmpty() && path.isEmpty()) throw onlyModelIdInHostedException(paramName); else return createModelReference(id, url, path, state); } @@ -106,11 +118,17 @@ public class ModelIdResolver { "Add a 'path' or 'url' to deploy this outside Vespa Cloud"); } - private static String modelIdToUrl(String valueName, String modelId) { + private static String modelIdToUrl(String valueName, String modelId, Set<String> requiredTags) { if ( ! providedModels.containsKey(modelId)) throw new IllegalArgumentException("Unknown model id '" + modelId + "' on '" + valueName + "'. Available models are [" + providedModels.keySet().stream().sorted().collect(Collectors.joining(", ")) + "]"); - return providedModels.get(modelId); + var providedModel = providedModels.get(modelId); + if (!providedModel.tags().containsAll(requiredTags)) { + throw new IllegalArgumentException( + "Model '%s' on '%s' has tags %s but are missing required tags %s" + .formatted(modelId, valueName, providedModel.tags(), requiredTags)); + } + return providedModel.uri().toString(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 81f071ba033..a79bc14db52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -138,7 +138,7 @@ public class IndexedSearchCluster extends SearchCluster for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), deployState)); + new DerivedConfiguration(spec.fullSchema(), deployState, false)); documentDbs.add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 3d48432fbe7..f25ca7e4452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -84,7 +84,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.derivedConfig = new DerivedConfiguration(schema, deployState); + this.derivedConfig = new DerivedConfiguration(schema, deployState, true); } @Override diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index aef91e34239..f298293ed57 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -181,6 +181,7 @@ TOKEN : | < GRAM: "gram" > | < GRAM_SIZE: "gram-size" > | < MAX_LENGTH: "max-length" > +| < MAX_OCCURRENCES: "max-occurrences" > | < PREFIX: "prefix" > | < SUBSTRING: "substring" > | < SUFFIX: "suffix" > @@ -1361,7 +1362,7 @@ void matchType(ParsedMatchSettings matchInfo) : { } */ void matchItem(ParsedMatchSettings matchInfo) : { } { - ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) ) + ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | maxTermOccurrences(matchInfo)) } void exactTerminator(ParsedMatchSettings matchInfo) : @@ -1396,6 +1397,16 @@ void matchSize(ParsedMatchSettings matchInfo) : } } +void maxTermOccurrences(ParsedMatchSettings matchInfo) : +{ + int maxTermOccurrences; +} +{ + <MAX_OCCURRENCES> <COLON> maxTermOccurrences = integer() { + matchInfo.setMaxTermOccurrences(maxTermOccurrences); + } +} + /** * Consumes a rank statement of a field element. * diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 2f3b10742f3..919253977ca 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -80,7 +80,7 @@ ComponentDefinition = TypedComponentDefinition = attribute id { xsd:Name } & - (HuggingFaceEmbedder | HuggingFaceTokenizer | BertBaseEmbedder | ColBertEmbedder) & + (HuggingFaceEmbedder | HuggingFaceTokenizer | BertBaseEmbedder | ColBertEmbedder | SpladeEmbedder ) & GenericConfig* & Component* @@ -97,6 +97,18 @@ HuggingFaceEmbedder = OnnxModelExecutionParams & EmbedderPoolingStrategy +SpladeEmbedder = + attribute type { "splade-embedder" } & + element transformer-model { ModelReference } & + element tokenizer-model { ModelReference }? & + element max-tokens { xsd:positiveInteger }? & + element transformer-input-ids { xsd:string }? & + element transformer-attention-mask { xsd:string }? & + element transformer-token-type-ids { xsd:string }? & + element transformer-output { xsd:string }? & + element term-score-threshold { xsd:double }? & + OnnxModelExecutionParams + HuggingFaceTokenizer = attribute type { "hugging-face-tokenizer" } & element model { attribute language { xsd:string }? & ModelReference }+ diff --git a/config-model/src/test/cfg/application/embed/services.xml b/config-model/src/test/cfg/application/embed/services.xml index 1840063d70d..59c29aefc6a 100644 --- a/config-model/src/test/cfg/application/embed/services.xml +++ b/config-model/src/test/cfg/application/embed/services.xml @@ -19,6 +19,21 @@ <pooling-strategy>mean</pooling-strategy> </component> + <component id="splade" type="splade-embedder"> + <transformer-model model-id="e5-base-v2" url="https://my/url/model.onnx"/> + <tokenizer-model model-id="e5-base-v2-vocab" path="app/tokenizer.json"/> + <max-tokens>1024</max-tokens> + <transformer-input-ids>my_input_ids</transformer-input-ids> + <transformer-attention-mask>my_attention_mask</transformer-attention-mask> + <transformer-token-type-ids>my_token_type_ids</transformer-token-type-ids> + <transformer-output>my_output</transformer-output> + <term-score-threshold>0.2</term-score-threshold> + <onnx-execution-mode>parallel</onnx-execution-mode> + <onnx-intraop-threads>10</onnx-intraop-threads> + <onnx-interop-threads>8</onnx-interop-threads> + <onnx-gpu-device>1</onnx-gpu-device> + </component> + <component id="hf-tokenizer" type="hugging-face-tokenizer"> <model language="no" model-id="multilingual-e5-base-vocab" url="https://my/url/tokenizer.json"/> </component> diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java index 3bb129a4c32..7686289f11c 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -52,7 +52,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { .deployLogger(logger) .rankProfileRegistry(builder.getRankProfileRegistry()) .queryProfiles(builder.getQueryProfileRegistry()) - .build()); + .build(), false); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java new file mode 100644 index 00000000000..09450fa8023 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java @@ -0,0 +1,55 @@ +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class IndexInfoTestCase { + private static final String F = "f"; + @Test + void testThatIndexingEnablesNormalizing() { + var cmds = createIndexCmds(false); + assertEquals(8, cmds.size()); + assertEquals(1, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count()); + } + @Test + void testThatStreamingDisablesNormalizing() { + var cmds = createIndexCmds(true); + assertEquals(7, cmds.size()); + assertEquals(0, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count()); + } + + private static List<IndexInfoConfig.Indexinfo.Command> createIndexCmds(boolean isStreaming) { + final String SD = "sda"; + String documentContent = "field " + F + " type string {indexing:index | summary}"; + var cfg = createIndexInfo(SD, documentContent, isStreaming); + assertEquals(SD, cfg.indexinfo(0).name()); + return cfg.indexinfo(0).command(); + } + + private static IndexInfoConfig createIndexInfo(String schemaName, String sdContent, boolean isStreaming) { + var model = createModel(schemaName, sdContent); + var schema = model.getSearchClusters().get(0).schemas().get(schemaName); + var indexInfo = new IndexInfo(schema.fullSchema(), isStreaming); + IndexInfoConfig.Builder builder = new IndexInfoConfig.Builder(); + indexInfo.getConfig(builder); + return builder.build(); + } + + private static VespaModel createModel(String schemaName, String sdContent) { + var builder = new DeployState.Builder(); + return new ApplicationPackageBuilder() + .addCluster(new ContentClusterBuilder().name("content").docTypes(List.of(DocType.index(schemaName)))) + .addSchemas(new SchemaBuilder().name(schemaName).content(sdContent).build()) + .buildCreator().create(builder); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java index b219d84f108..2beab3d5ea9 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java @@ -46,7 +46,7 @@ public class SchemaToDerivedConfigExporter { .deployLogger(logger) .rankProfileRegistry(builder.getRankProfileRegistry()) .queryProfiles(builder.getQueryProfileRegistry()) - .build()); + .build(), false); exportConfig(dirPath, derived, builder); } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index c014f77d59a..5a2dc218da7 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -120,6 +120,23 @@ public class SchemaParserTestCase { assertEquals("onnx(mymodel)", rp1.getGlobalPhaseExpression().get()); } + @Test + void maxOccurrencesCanBeParsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " field bar type string {", + " indexing: summary | index", + " match { max-occurrences: 11 }", + " }", + " }", + "}"); + ParsedSchema schema = parseString(input); + var field = schema.getDocument().getFields().get(0); + assertEquals("bar", field.name()); + assertEquals(11, field.matchSettings().getMaxTermOccurrences().get()); + } + void checkFileParses(String fileName) throws Exception { var schema = parseFile(fileName); assertNotNull(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index 87e91acfb67..c2cc28ea6b3 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -147,6 +147,15 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { createPredicateField("test", DataType.PREDICATE, "{ attribute; }", 2, OptionalLong.of(0L), OptionalLong.of(1023L))); } + @Test + void requireThatMaxTermOccurrencesIsPropagated() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTermOccurrences(10); + field.parseIndexingScript("{ summary | index }"); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-occurrences:10 | summary test | index test; }", + field); + } + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { assertEquals(expectedScript, processField(unprocessedField).toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java index 25053c536da..4e88753b732 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import static com.yahoo.vespa.model.application.validation.ValidationTester.expect; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -56,9 +57,8 @@ public class AccessControlFilterExcludeValidatorTest { MapConfigModelRegistry.createFromList(new ModelBuilderAddingAccessControlFilter()), deployState); - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> new AccessControlFilterExcludeValidator().validate(model, deployState)); - String expectedMessage = "Application cluster container-cluster-with-access-control excludes paths from access control, this is not allowed and should be removed."; - assertEquals(expectedMessage, exception.getMessage()); + expect(new AccessControlFilterExcludeValidator(), model, deployState, + "Application cluster container-cluster-with-access-control excludes paths from access control, this is not allowed and should be removed."); } @Test @@ -69,9 +69,8 @@ public class AccessControlFilterExcludeValidatorTest { MapConfigModelRegistry.createFromList(new ModelBuilderAddingAccessControlFilter()), deployState); - new AccessControlFilterExcludeValidator().validate(model, deployState); - String expectedMessage = "Application cluster container-cluster-with-access-control excludes paths from access control, this is not allowed and should be removed."; - assertTrue(logOutput.toString().contains(expectedMessage)); + ValidationTester.validate(new AccessControlFilterExcludeValidator(), model, deployState); + assertTrue(logOutput.toString().contains("Application cluster container-cluster-with-access-control excludes paths from access control, this is not allowed and should be removed.")); } @Test @@ -80,7 +79,7 @@ public class AccessControlFilterExcludeValidatorTest { VespaModel model = new VespaModel( MapConfigModelRegistry.createFromList(new ModelBuilderAddingAccessControlFilter()), deployState); - new AccessControlFilterExcludeValidator().validate(model, deployState); + ValidationTester.validate(new AccessControlFilterExcludeValidator(), model, deployState); } @Test @@ -90,7 +89,7 @@ public class AccessControlFilterExcludeValidatorTest { MapConfigModelRegistry.createFromList(new ModelBuilderAddingAccessControlFilter()), deployState); - new AccessControlFilterExcludeValidator().validate(model, deployState); + ValidationTester.validate(new AccessControlFilterExcludeValidator(), model, deployState); } private static DeployState createDeployState(Zone zone, StringBuffer buffer, boolean allowExcludes) { @@ -112,6 +111,6 @@ public class AccessControlFilterExcludeValidatorTest { Cloud.Builder cloudBuilder = Cloud.builder().name(cloudName); if (cloudName == CloudName.AWS) cloudBuilder.account(CloudAccount.from("123456789012")); return new Zone(cloudBuilder.build(), systemName, Environment.prod, RegionName.defaultName()); - } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java index aa42fbbf827..1bc59e118d4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java @@ -39,7 +39,7 @@ public class AccessControlFilterValidatorTest { VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); try { - new AccessControlFilterValidator().validate(model, deployState); + ValidationTester.validate(new AccessControlFilterValidator(), model, deployState); fail(); } catch (IllegalArgumentException e) { assertEquals("The 'access-control' feature is not available in open-source Vespa.", e.getMessage()); @@ -53,7 +53,7 @@ public class AccessControlFilterValidatorTest { MapConfigModelRegistry.createFromList(new ModelBuilderAddingAccessControlFilter()), deployState); - new AccessControlFilterValidator().validate(model, deployState); + ValidationTester.validate(new AccessControlFilterValidator(), model, deployState); } private static DeployState createDeployState() { @@ -61,4 +61,5 @@ public class AccessControlFilterValidatorTest { .applicationPackage(new MockApplicationPackage.Builder().withServices(SERVICES_XML).build()) .build(); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index f41cc266db3..1aca0c2fe47 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.application.validation.AbstractBundleValidator.JarContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -29,7 +30,7 @@ public class BundleValidatorTest { // Valid jar file JarFile ok = createTemporaryJarFile(tempDir, "ok"); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(DeployState.createTestState(), ok); + bundleValidator.validateJarFile(contextOf(DeployState.createTestState()), ok); // No manifest validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); @@ -39,7 +40,7 @@ public class BundleValidatorTest { try { JarFile jarFile = createTemporaryJarFile(tempDir, jarName); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(DeployState.createTestState(), jarFile); + bundleValidator.validateJarFile(contextOf(DeployState.createTestState()), jarFile); assert (false); } catch (IllegalArgumentException e) { assertEquals(exceptionMessage, e.getMessage()); @@ -52,7 +53,7 @@ public class BundleValidatorTest { DeployState state = createDeployState(buffer); JarFile jarFile = createTemporaryJarFile(tempDir, "snapshot_bundle"); - new BundleValidator().validateJarFile(state, jarFile); + new BundleValidator().validateJarFile(contextOf(state), jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @@ -62,7 +63,7 @@ public class BundleValidatorTest { DeployState state = createDeployState(buffer); BundleValidator validator = new BundleValidator(); JarFile jarFile = createTemporaryJarFile(tempDir, "import-warnings"); - validator.validateJarFile(state, jarFile); + validator.validateJarFile(contextOf(state), jarFile); String output = buffer.toString(); assertTrue(output .contains("JAR file 'import-warnings.jar' imports the packages [org.json] from 'org.json:json'. \n" + @@ -123,5 +124,12 @@ public class BundleValidatorTest { List.of("org.json", "version", "[0.0.0,1)", "org.eclipse.jetty.client.api", "version", "[9.4.46,10)")); } + private static JarContext contextOf(DeployState state) { + return new JarContext() { + @Override public void illegal(String error) { throw new IllegalArgumentException(error); } + @Override public void illegal(String error, Throwable cause) { throw new IllegalArgumentException(error, cause); } + @Override public DeployState deployState() { return state; } + }; + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java new file mode 100644 index 00000000000..6fbca76ccbc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import org.junit.jupiter.api.Test; + +import java.security.cert.X509Certificate; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bjorncs + */ +class CloudClientsValidatorTest { + + @Test + void logs_deployment_warning_on_certificate_with_empty_sequence_of_extensions() { + // Test should fail on BouncyCastle 1.77 or later + + var logger = new DeployLoggerStub(); + var state = new DeployState.Builder().deployLogger(logger).build(); + var cert = readTestCertificate("cert-with-empty-sequence-of-extensions.pem"); + CloudClientsValidator.validateCertificate("default", "my-feed-client", cert, + (msg, cause) -> { throw new IllegalArgumentException(msg, cause); }, + state); + var expected = "Client **my-feed-client** defined for cluster **default** contains an invalid certificate: " + + "The certificate's ASN.1 structure contains an empty sequence of extensions, " + + "which is a violation of the ASN.1 specification. " + + "Please update the application package with a new certificate, " + + "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. " + + "Such certificate will no longer be accepted in near future."; + assertEquals(expected, logger.getLast().message); + } + + @Test + void accepts_valid_certificate() { + var logger = new DeployLoggerStub(); + var state = new DeployState.Builder().deployLogger(logger).build(); + var cert = readTestCertificate("valid-cert.pem"); + assertDoesNotThrow(() -> CloudClientsValidator.validateCertificate("default", "my-feed-client", cert, + (msg, cause) -> { throw new IllegalArgumentException(msg, cause); }, + state)); + assertEquals(0, logger.entries.size()); + } + + private static X509Certificate readTestCertificate(String filename) { + return X509CertificateUtils.fromPem(new String(uncheck( + () -> CloudClientsValidatorTest.class.getResourceAsStream( + "/cloud-clients-validator/%s".formatted(filename)).readAllBytes()))); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java index 8acbf00a5a3..80ef81ee6d7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java @@ -71,7 +71,7 @@ public class CloudDataPlaneFilterValidatorTest { certFile2, List.of(createCertificate("bar")))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new CloudDataPlaneFilterValidator().validate(model, deployState); + ValidationTester.validate(new CloudDataPlaneFilterValidator(), model, deployState); } @Test @@ -100,11 +100,8 @@ public class CloudDataPlaneFilterValidatorTest { certFile2, List.of(certificate))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> - new CloudDataPlaneFilterValidator().validate(model, deployState)); - assertEquals( - "Duplicate certificate(s) detected in files: [%s, %s]. Certificate subject of duplicates: [%s]".formatted(certFile1, certFile2, certificate.getSubjectX500Principal().getName()), - illegalArgumentException.getMessage()); + ValidationTester.expect(new CloudDataPlaneFilterValidator(), model, deployState, + "Duplicate certificate(s) detected in files: [%s, %s]. Certificate subject of duplicates: [%s]".formatted(certFile1, certFile2, certificate.getSubjectX500Principal().getName())); } @Test @@ -127,11 +124,8 @@ public class CloudDataPlaneFilterValidatorTest { Map.of(certFile1, List.of(certificate, certificate))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> - new CloudDataPlaneFilterValidator().validate(model, deployState)); - assertEquals( - "Duplicate certificate(s) detected in files: [%s]. Certificate subject of duplicates: [%s]".formatted(certFile1, certificate.getSubjectX500Principal().getName()), - illegalArgumentException.getMessage()); + ValidationTester.expect(new CloudDataPlaneFilterValidator(), model, deployState, + "Duplicate certificate(s) detected in files: [%s]. Certificate subject of duplicates: [%s]".formatted(certFile1, certificate.getSubjectX500Principal().getName())); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java index 58aa0e8625e..3be1cbd44e3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java @@ -104,7 +104,7 @@ class CloudHttpConnectorValidatorTest { .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .build(); var model = new VespaModel(new NullConfigModelRegistry(), state); - new CloudHttpConnectorValidator().validate(model, state); + ValidationTester.validate(new CloudHttpConnectorValidator(), model, state); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java index 2aa678fd34b..ac5a08b1394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java @@ -67,7 +67,7 @@ class CloudUserFilterValidatorTest { .properties(new TestProperties().setHostedVespa(isHosted).setAllowUserFilters(false)) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new CloudUserFilterValidator().validate(model, deployState); + ValidationTester.validate(new CloudUserFilterValidator(), model, deployState); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java index 43c51bea04a..b6484049eaf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java @@ -61,7 +61,7 @@ public class ContainerInCloudValidatorTest { } DeployState deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new ContainerInCloudValidator().validate(model, deployState); + ValidationTester.validate(new ContainerInCloudValidator(), model, deployState); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java index 4e388df3ef8..c9b014d9301 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java @@ -63,7 +63,7 @@ public class DeploymentSpecValidatorTest { try { var deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new DeploymentSpecValidator().validate(model, deployState); + ValidationTester.validate(new DeploymentSpecValidator(), model, deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { assertEquals(message, e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index 821ad1be8fa..3b6a559ce31 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -48,7 +48,7 @@ public class EndpointCertificateSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(EndpointCertificateSecrets.missing(1))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + ValidationTester.validate(new EndpointCertificateSecretsValidator(), model, deployState); }); assertTrue(exception.getMessage().contains("TLS enabled, but could not yet retrieve certificate version 1 for application default:default:default")); } @@ -58,7 +58,7 @@ public class EndpointCertificateSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new EndpointCertificateSecrets("cert", "key"))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + ValidationTester.validate(new EndpointCertificateSecretsValidator(), model, deployState); } @Test @@ -66,7 +66,7 @@ public class EndpointCertificateSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.empty()); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); + ValidationTester.validate(new EndpointCertificateSecretsValidator(), model, deployState); } private static DeployState deployState(String servicesXml, String deploymentXml, Optional<EndpointCertificateSecrets> endpointCertificateSecretsSecrets) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java index bcec73432b3..190f68e6956 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java @@ -43,6 +43,7 @@ public class InfrastructureDeploymentValidatorTest { var model = new VespaModel(new NullConfigModelRegistry(), deployState); var validator = new InfrastructureDeploymentValidator(); - validator.validate(model, deployState); + ValidationTester.validate(validator, model, deployState); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 31e4c661151..a53ef233746 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; @@ -38,27 +39,25 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_jvm_percentage() throws IOException, SAXException { - var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); + var deployState = createDeployState(9, 7L * 1024 * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); - var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); - String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 15%). Estimated cost of ONNX models is 7.00GB"; - assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, + "Allocated percentage of memory of JVM in cluster 'container' is too low (12% < 15%). Estimated cost of ONNX models is 7.00GB"); } @Test void fails_on_too_low_heap_size() throws IOException, SAXException { var deployState = createDeployState(2.2, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); - var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); - String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."; - assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, + "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test void accepts_adequate_heap_size() throws IOException, SAXException { var deployState = createDeployState(8, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); - assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + assertDoesNotThrow(() -> ValidationTester.validate(new JvmHeapSizeValidator(), model, deployState)); } @Test @@ -80,7 +79,7 @@ class JvmHeapSizeValidatorTest { </services>"""; var deployState = createDeployState(servicesXml, 2, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); - assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + assertDoesNotThrow(() -> ValidationTester.validate(new JvmHeapSizeValidator(), model, deployState)); } private static DeployState createDeployState(String servicesXml, double nodeGb, long modelCostBytes) { @@ -121,6 +120,7 @@ class JvmHeapSizeValidatorTest { ModelCostDummy(long modelCost) { this.modelCost = modelCost; } @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } + @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId, ClusterSpec.Id clusterId) { return this; } @Override public Map<String, ModelInfo> models() { return Map.of(); } @Override public void setRestartOnDeploy() {} @Override public boolean restartOnDeploy() { return false;} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java index c68599f4595..19be886d3e5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.application.validation.AbstractBundleValidator.JarContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -13,6 +15,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.fail; /** * @author gjoranv @@ -29,7 +32,11 @@ public class PublicApiBundleValidatorTest { var jarFile = BundleValidatorTest.createTemporaryJarFile(tempDir, "non-public-api"); var validator = new PublicApiBundleValidator(); - validator.validateJarFile(deployState, jarFile); + validator.validateJarFile(new JarContext() { + @Override public void illegal(String error) { fail(); } + @Override public void illegal(String error, Throwable cause) { fail(); } + @Override public DeployState deployState() { return deployState; } + }, jarFile); String output = outputBuf.toString(); assertThat(output, containsString("uses non-public Vespa APIs: [")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java index ae23b3b722d..9d53c5af61c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java @@ -48,7 +48,7 @@ public class SecretStoreValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(true)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new SecretStoreValidator().validate(model, deployState); + ValidationTester.validate(new SecretStoreValidator(), model, deployState); } @Test @@ -58,7 +58,7 @@ public class SecretStoreValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(false)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new SecretStoreValidator().validate(model, deployState); + ValidationTester.validate(new SecretStoreValidator(), model, deployState); }); assertTrue(exception.getMessage().contains("Container cluster 'default' uses a secret store, so an Athenz domain and" + @@ -74,7 +74,7 @@ public class SecretStoreValidatorTest { DeployState deployState = deployState(servicesXml, deploymentXml(false)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new SecretStoreValidator().validate(model, deployState); + ValidationTester.validate(new SecretStoreValidator(), model, deployState); } private static DeployState deployState(String servicesXml, String deploymentXml) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java index 9a2f9fadac6..92c2b5276cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java @@ -107,7 +107,7 @@ public class UriBindingsValidatorTest { .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new UriBindingsValidator().validate(model, deployState); + ValidationTester.validate(new UriBindingsValidator(), model, deployState); } private static String createServicesXmlWithHandler(String handlerBinding) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java index 837de946e36..d4a324901e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -81,7 +81,7 @@ public class UrlConfigValidatorTest { .build(); DeployState deployState = createDeployState(app, systemName); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new UrlConfigValidator().validate(model, deployState); + ValidationTester.validate(new UrlConfigValidator(), model, deployState); } private static DeployState createDeployState(ApplicationPackage app, SystemName systemName) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 8dc07d8857d..29279635918 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -16,12 +16,15 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.Execution; +import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -30,6 +33,8 @@ import java.util.stream.Stream; import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SCHEMA; import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SCHEMA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bratseth @@ -114,4 +119,27 @@ public class ValidationTester { return s.replaceAll("\\d", "-"); } + public static void expect(Validator validator, VespaModel model, DeployState deployState, String... expectedMessages) { + Execution execution = new Execution(model, deployState); + validator.validate(execution); + assertTrue( execution.errors().stream().allMatch(error -> Arrays.stream(expectedMessages).anyMatch(error::contains)) + && Arrays.stream(expectedMessages).allMatch(expected -> execution.errors().stream().anyMatch(error -> error.contains(expected))), + "Expected errors: " + Arrays.toString(expectedMessages) + "\nActual errors: " + execution.errors()); + } + + /** Runs validation, and throws on illegalities. */ + public static void validate(Validator validator, VespaModel model, DeployState deployState) { + Execution execution = new Execution(model, deployState); + validator.validate(execution); + execution.throwIfFailed(); + } + + /** Runs validation and returns the resulting config chance actions, without checking whether they're currently allowed; or throws on illegalities. */ + public static List<ConfigChangeAction> validateChanges(ChangeValidator validator, VespaModel model, DeployState deployState) { + Execution execution = new Execution(model, deployState); + validator.validate(execution); + if ( ! execution.errors().isEmpty()) execution.throwIfFailed(); + return execution.actions(); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java index bc36b800bfb..6b5db3b081f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java @@ -41,22 +41,22 @@ public class CertificateRemovalChangeValidatorTest { CertificateRemovalChangeValidator validator = new CertificateRemovalChangeValidator(); // Adding certs -> ok - validator.validateClients("clusterId", List.of(c1, c2), List.of(c1, c2, c3), new DeployState.Builder().now(now).build()); + validator.validateClients("clusterId", List.of(c1, c2), List.of(c1, c2, c3), (id, msg) -> ValidationOverrides.empty.invalid(id, msg, now)); // Removing certs -> fails assertThrows(ValidationOverrides.ValidationException.class, () ->validator.validateClients("clusterId", List.of(c1, c2, c3), List.of(c1, c3), - new DeployState.Builder().now(now).build())); + (id, msg) -> ValidationOverrides.empty.invalid(id, msg, now))); // Removing certs with validationoverrides -> ok validator.validateClients("clusterId", List.of(c1, c2, c3), List.of(c1, c3), - new DeployState.Builder().now(now).validationOverrides(ValidationOverrides.fromXml(validationOverrides)).build()); + (id, msg) -> ValidationOverrides.fromXml(validationOverrides).invalid(id, msg, now)); // Adding and removing internal certs are ok: validator.validateClients("clusterId", List.of(c1, c2), List.of(c1, c2, internal), - new DeployState.Builder().build()); + (id, msg) -> ValidationOverrides.empty.invalid(id, msg, now)); validator.validateClients("clusterId", List.of(c1, c2, internal), List.of(c1, c2), - new DeployState.Builder().now(now).build()); + (id, msg) -> ValidationOverrides.empty.invalid(id, msg, now)); } static X509Certificate certificate(String cn) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 081e10ecea6..c9703fc34af 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -7,9 +7,11 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import static java.util.Comparator.comparing; import static org.junit.jupiter.api.Assertions.assertEquals; public class ConfigChangeTestUtils { @@ -60,8 +62,8 @@ public class ConfigChangeTestUtils { public static void assertEqualActions(List<ConfigChangeAction> exp, List<ConfigChangeAction> act) { var mutableExp = new ArrayList<>(exp); var mutableAct = new ArrayList<>(act); - mutableExp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - mutableAct.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + mutableExp.sort(comparing(ConfigChangeAction::getMessage)); + mutableAct.sort(comparing(ConfigChangeAction::getMessage)); assertEquals(mutableExp, mutableAct); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index a41b538d3ca..f68a1da7dfb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -1,16 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.test.AnotherrestartConfig; import com.yahoo.config.ConfigInstance; -import com.yahoo.test.RestartConfig; -import com.yahoo.test.SimpletypesConfig; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.test.AnotherrestartConfig; +import com.yahoo.test.RestartConfig; +import com.yahoo.test.SimpletypesConfig; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; @@ -25,7 +24,9 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Testing the validator on both a stub model and a real-life Vespa model. @@ -152,11 +153,6 @@ public class ConfigValueChangeValidatorTest { assertEmptyLog(); } - private List<ConfigChangeAction> getConfigChanges(VespaModel currentModel, VespaModel nextModel) { - ConfigValueChangeValidator validator = new ConfigValueChangeValidator(); - return validator.validate(currentModel, nextModel, new DeployState.Builder().deployLogger(logger).build()); - } - private List<ConfigChangeAction> getConfigChanges(AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { ConfigValueChangeValidator validator = new ConfigValueChangeValidator(); @@ -245,8 +241,7 @@ public class ConfigValueChangeValidatorTest { setHostResource(new HostResource(new Host(null, "localhost"))); } - @Override - public int getPortCount() { + @Override public int getPortCount() { return 0; } @@ -262,8 +257,7 @@ public class ConfigValueChangeValidatorTest { this.value = value; } - @Override - public void getConfig(RestartConfig.Builder builder) { + @Override public void getConfig(RestartConfig.Builder builder) { builder.value(value); } @@ -283,8 +277,7 @@ public class ConfigValueChangeValidatorTest { this.anotherValue = anotherValue; } - @Override - public void getConfig(AnotherrestartConfig.Builder builder) { + @Override public void getConfig(AnotherrestartConfig.Builder builder) { builder.anothervalue(anotherValue); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java index ac59ca58cb5..a7485b177d3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; @@ -67,7 +68,7 @@ public class ContainerRestartValidatorTest { } private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { - return new ContainerRestartValidator().validate(current, next, new DeployState.Builder().build()); + return ValidationTester.validateChanges(new ContainerRestartValidator(), next, new DeployState.Builder().previousModel(current).build()); } private static VespaModel createModel(boolean restartOnDeploy, boolean alwaysRestart) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java index cdc80754194..f4123a72e1f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; @@ -14,7 +15,7 @@ import static org.junit.jupiter.api.Assertions.*; public class GlobalDocumentChangeValidatorTest { @Test - void testChangGlobalAttribute() { + void testChangeGlobalAttribute() { testChangeGlobalAttribute(true, false, false, null); testChangeGlobalAttribute(true, true, true, null); testChangeGlobalAttribute(false, false, true, null); @@ -27,14 +28,16 @@ public class GlobalDocumentChangeValidatorTest { ValidationTester tester = new ValidationTester(); VespaModel oldModel = tester.deploy(null, getServices(oldGlobal), Environment.prod, validationOverrides, "default.indexing").getFirst(); try { - tester.deploy(oldModel, getServices(newGlobal), Environment.prod, validationOverrides, "default.indexing").getSecond(); + var actions = tester.deploy(oldModel, getServices(newGlobal), Environment.prod, validationOverrides, "default.indexing").getSecond(); assertTrue(allowed); - } catch (IllegalStateException e) { + assertEquals(validationOverrides == null ? 0 : 1, actions.size()); + if (validationOverrides != null) assertEquals(ClusterSpec.Id.from("default"), actions.get(0).clusterId()); + } catch (IllegalArgumentException e) { assertFalse(allowed); - assertEquals("Document type music in cluster default changed global from " + oldGlobal + " to " + newGlobal + ". " + - "Add validation override 'global-document-change' to force this change through. " + - "First, stop services on all content nodes. Then, deploy with validation override. Finally, start services on all content nodes.", - e.getMessage()); + assertEquals("global-document-change: Document type music in cluster default changed global from " + oldGlobal + " to " + newGlobal + ". " + + "To handle this change, first stop services on all content nodes. Then, deploy with validation override. Finally, start services on all content nodes. " + + "To allow this add <allow until='yyyy-mm-dd'>global-document-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", + e.getMessage()); } } @@ -52,8 +55,10 @@ public class GlobalDocumentChangeValidatorTest { } private static final String globalDocumentValidationOverrides = - "<validation-overrides>\n" + - " <allow until='2000-01-14' comment='test override'>global-document-change</allow>\n" + - "</validation-overrides>\n"; + """ + <validation-overrides> + <allow until='2000-01-14' comment='test override'>global-document-change</allow> + </validation-overrides> + """; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSchemaClusterChangeValidatorTest.java index fb4e9f1a00b..3555fc21471 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSchemaClusterChangeValidatorTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.SchemaBuilder; @@ -70,8 +71,7 @@ public class IndexedSchemaClusterChangeValidatorTest { } private List<ConfigChangeAction> validate() { - return normalizeServicesInActions(validator.validate(currentModel, nextModel, - new DeployState.Builder().build())); + return normalizeServicesInActions(ValidationTester.validateChanges(validator, nextModel, new DeployState.Builder().previousModel(currentModel).build())); } public void assertValidation() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java index afa36ac271e..b2439651cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; @@ -58,7 +59,7 @@ public class NodeResourceChangeValidatorTest { } private List<ConfigChangeAction> validate(VespaModel current, VespaModel next) { - return new NodeResourceChangeValidator().validate(current, next, new DeployState.Builder().build()); + return ValidationTester.validateChanges(new NodeResourceChangeValidator(), next, new DeployState.Builder().previousModel(current).build()); } private static VespaModel model(int mem1, int mem2, int mem3, int mem4) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 13389689de5..67e8a4d512e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; @@ -71,7 +72,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { } private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { - return new RestartOnDeployForOnnxModelChangesValidator().validate(current, next, deployStateBuilder().build()); + return ValidationTester.validateChanges(new RestartOnDeployForOnnxModelChangesValidator(), next, deployStateBuilder().previousModel(current).build()); } private static OnnxModelCost onnxModelCost() { @@ -79,7 +80,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { } private static OnnxModelCost onnxModelCost(long estimatedCost, long hash) { - return (appPkg, applicationId) -> new OnnxModelCost.Calculator() { + return (appPkg, applicationId, clusterId) -> new OnnxModelCost.Calculator() { private final Map<String, OnnxModelCost.ModelInfo> models = new HashMap<>(); private boolean restartOnDeploy = false; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java index 8db9d39534d..ee64ceb6969 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.DocType; @@ -69,8 +70,8 @@ public class StreamingSchemaClusterChangeValidatorTest { } public List<ConfigChangeAction> validate() { - return normalizeServicesInActions(validator.validate(currentModel, nextModel, - new DeployState.Builder().build())); + return normalizeServicesInActions(ValidationTester.validateChanges(validator, nextModel, + new DeployState.Builder().previousModel(currentModel).build())); } public void assertValidation() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java index 138bef3ae73..2532a5be863 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.embedding.ColBertEmbedderConfig; +import com.yahoo.embedding.SpladeEmbedderConfig; import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.language.huggingface.config.HuggingFaceTokenizerConfig; import com.yahoo.path.Path; @@ -24,6 +25,7 @@ import com.yahoo.vespa.model.container.component.BertEmbedder; import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; +import com.yahoo.vespa.model.container.component.SpladeEmbedder; import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import com.yahoo.yolean.Exceptions; @@ -101,6 +103,23 @@ public class EmbedderTestCase { assertEquals(-1, tokenizerCfg.maxLength()); } + @Test + void spladeEmbedder_selfhosted() throws Exception { + var model = loadModel(Path.fromString("src/test/cfg/application/embed/"), false); + var cluster = model.getContainerClusters().get("container"); + var embedderCfg = assertSpladeEmbedderComponentPresent(cluster); + + assertEquals("my_input_ids", embedderCfg.transformerInputIds()); + assertEquals("https://my/url/model.onnx", modelReference(embedderCfg, "transformerModel").url().orElseThrow().value()); + assertEquals(0.2, embedderCfg.termScoreThreshold()); + assertEquals(1024, embedderCfg.transformerMaxTokens()); + + var tokenizerCfg = assertHuggingfaceTokenizerComponentPresent(cluster); + assertEquals("https://my/url/tokenizer.json", modelReference(tokenizerCfg.model().get(0), "path").url().orElseThrow().value()); + assertEquals(-1, tokenizerCfg.maxLength()); + } + + @Test void colBertEmbedder_selfhosted() throws Exception { var model = loadModel(Path.fromString("src/test/cfg/application/embed/"), false); var cluster = model.getContainerClusters().get("container"); @@ -113,6 +132,7 @@ public class EmbedderTestCase { assertEquals(-1, tokenizerCfg.maxLength()); } + @Test void colBertEmbedder_hosted() throws Exception { var model = loadModel(Path.fromString("src/test/cfg/application/embed/"), true); var cluster = model.getContainerClusters().get("container"); @@ -266,13 +286,21 @@ public class EmbedderTestCase { } private static ColBertEmbedderConfig assertColBertEmbedderComponentPresent(ApplicationContainerCluster cluster) { - var colbert = (ColBertEmbedder) cluster.getComponentsMap().get(new ComponentId("colbert-embedder")); + var colbert = (ColBertEmbedder) cluster.getComponentsMap().get(new ComponentId("colbert")); assertEquals("ai.vespa.embedding.ColBertEmbedder", colbert.getClassId().getName()); var cfgBuilder = new ColBertEmbedderConfig.Builder(); colbert.getConfig(cfgBuilder); return cfgBuilder.build(); } + private static SpladeEmbedderConfig assertSpladeEmbedderComponentPresent(ApplicationContainerCluster cluster) { + var splade = (SpladeEmbedder) cluster.getComponentsMap().get(new ComponentId("splade")); + assertEquals("ai.vespa.embedding.SpladeEmbedder", splade.getClassId().getName()); + var cfgBuilder = new SpladeEmbedderConfig.Builder(); + splade.getConfig(cfgBuilder); + return cfgBuilder.build(); + } + private static BertBaseEmbedderConfig assertBertEmbedderComponentPresent(ApplicationContainerCluster cluster) { var bertEmbedder = (BertEmbedder) cluster.getComponentsMap().get(new ComponentId("bert-embedder")); assertEquals("ai.vespa.embedding.BertBaseEmbedder", bertEmbedder.getClassId().getName()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ModelIdResolverTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ModelIdResolverTest.java new file mode 100644 index 00000000000..409c3ac833a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ModelIdResolverTest.java @@ -0,0 +1,35 @@ +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.HF_TOKENIZER; +import static com.yahoo.vespa.model.container.xml.ModelIdResolver.ONNX_MODEL; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author bjorncs + */ +class ModelIdResolverTest { + + @Test + void throws_on_known_model_with_missing_tags() { + var state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)).build(); + var e = assertThrows(IllegalArgumentException.class, () -> + ModelIdResolver.resolveToModelReference( + "param", Optional.of("minilm-l6-v2"), Optional.empty(), Optional.empty(), Set.of(HF_TOKENIZER), state)); + var expectedMsg = "Model 'minilm-l6-v2' on 'param' has tags [onnx-model] but are missing required tags [huggingface-tokenizer]"; + assertEquals(expectedMsg, e.getMessage()); + + assertDoesNotThrow( + () -> ModelIdResolver.resolveToModelReference( + "param", Optional.of("minilm-l6-v2"), Optional.empty(), Optional.empty(), Set.of(ONNX_MODEL), state)); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java index 6defca17bcb..304f3dc426f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content.utils; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; diff --git a/config-model/src/test/resources/cloud-clients-validator/cert-with-empty-sequence-of-extensions.pem b/config-model/src/test/resources/cloud-clients-validator/cert-with-empty-sequence-of-extensions.pem new file mode 100644 index 00000000000..1942c12b28e --- /dev/null +++ b/config-model/src/test/resources/cloud-clients-validator/cert-with-empty-sequence-of-extensions.pem @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBPDCB46ADAgECAhEAhdEB3eHnsQxTdcYcClVpkzAKBggqhkjOPQQDAjAeMRww +GgYDVQQDExNjbG91ZC52ZXNwYS5leGFtcGxlMB4XDTIyMTIyMTE4NTg0MloXDTMy +MTIxODE4NTg0MlowHjEcMBoGA1UEAxMTY2xvdWQudmVzcGEuZXhhbXBsZTBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABOPzpGlb+4HvgsRT5Ic6gmzYqAE2GQrgfi5z +txf8yzoi5YqgEG6utFhjleQ5bUusDhMtrfOJoBL5VZxrQccmwsCjAjAAMAoGCCqG +SM49BAMCA0gAMEUCIQCuNXMk5lsb9lF2IloYZB2wAHme/xAOyQ2arWzZf6BH2wIg +dEsmbGhel9MLlfPVQjeUwCJha/XD7xfWW6IaL+hI5TQ= +-----END CERTIFICATE----- diff --git a/config-model/src/test/resources/cloud-clients-validator/valid-cert.pem b/config-model/src/test/resources/cloud-clients-validator/valid-cert.pem new file mode 100644 index 00000000000..aebec508772 --- /dev/null +++ b/config-model/src/test/resources/cloud-clients-validator/valid-cert.pem @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBNzCB3qADAgECAhB00FXOPRoixiJghCsT2FVOMAoGCCqGSM49BAMCMB4xHDAa +BgNVBAMTE2Nsb3VkLnZlc3BhLmV4YW1wbGUwHhcNMjQwMTA0MTM0MTMwWhcNMzQw +MTAxMTM0MTMwWjAeMRwwGgYDVQQDExNjbG91ZC52ZXNwYS5leGFtcGxlMFkwEwYH +KoZIzj0CAQYIKoZIzj0DAQcDQgAEts/fYf1H+aOW4xZHtcxX2YvMWojzU4HvHw1b +9Zc+7OoUcoqv9dTZMVaYj3J8Z3A73wNn5rhjPrI4sKtI5KN6sjAKBggqhkjOPQQD +AgNIADBFAiEAqgs4QouJOf6ny48o5c6EZSTB3+iNyZr+23JXKwnYuUkCIFRtE736 +BJ5KdCPpI4jS611HgeLLlJmgF2524Gz4EpjH +-----END CERTIFICATE----- |