diff options
Diffstat (limited to 'config-application-package')
11 files changed, 197 insertions, 35 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java index cd68d214d3d..4268e4f835e 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java @@ -41,7 +41,7 @@ class IncludeProcessor implements PreProcessor { Element elem = (Element) list.item(0); Element parent = (Element) elem.getParentNode(); String filename = elem.getAttribute("file"); - boolean required = elem.hasAttribute("required") ? Boolean.parseBoolean(elem.getAttribute("required")) : true; + boolean required = ! elem.hasAttribute("required") || Boolean.parseBoolean(elem.getAttribute("required")); File file = new File(currentFolder, filename); Document subFile = IncludeProcessor.parseIncludeFile(file, parent.getTagName(), required); @@ -76,4 +76,5 @@ class IncludeProcessor implements PreProcessor { w.append(endTag); return XML.getDocument(new StringReader(w.toString())); } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index f3da285f524..32e9aec56cb 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; * @since 5.22 */ class OverrideProcessor implements PreProcessor { + private static final Logger log = Logger.getLogger(OverrideProcessor.class.getName()); private final Environment environment; @@ -140,6 +141,9 @@ class OverrideProcessor implements PreProcessor { } } + if (bestMatch > 1) // there was a region/environment specific overriode + doElementSpecificProcessingOnOverride(bestMatchElement); + // Remove elements not specific for (Element child : children) { if (child != bestMatchElement) { @@ -148,6 +152,14 @@ class OverrideProcessor implements PreProcessor { } } + /** Called on each element which is selected by matching some override condition */ + private void doElementSpecificProcessingOnOverride(Element element) { + // if node capacity is specified explicitly for some evn/region we should require that capacity + if ( element.getTagName().equals("nodes")) + if (element.getChildNodes().getLength() == 0) // specifies capacity, not a list of nodes + element.setAttribute("required", "true"); + } + /** * Retains all elements where at least one element is overridden. Removes non-overridden elements from map. */ diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java index 4e08e514504..b70a5054563 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java @@ -25,6 +25,7 @@ import java.util.List; * @since 5.22 */ public class XmlPreProcessor { + final static String deployNamespace = "xmlns:deploy"; final static String deployNamespaceUri = "vespa"; final static String preprocessNamespace = "xmlns:preprocess"; @@ -68,4 +69,5 @@ public class XmlPreProcessor { chain.add(new PropertiesProcessor()); return chain; } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 393bd1c2de7..06ecede09a5 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -7,12 +7,10 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.Version; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -24,7 +22,6 @@ import java.util.Optional; public class ApplicationPackageXmlFilesValidator { private final AppSubDirs appDirs; - private final DeployLogger logger; private final Optional<Version> vespaVersion; private static final FilenameFilter xmlFilter = new FilenameFilter() { @@ -34,31 +31,32 @@ public class ApplicationPackageXmlFilesValidator { } }; - public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, DeployLogger logger, Optional<Version> vespaVersion) { + + public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, Optional<Version> vespaVersion) { this.appDirs = appDirs; - this.logger = logger; this.vespaVersion = vespaVersion; } - public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, DeployLogger logger, Optional<Version> vespaVersion) { - return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), logger, vespaVersion); + // TODO: Remove when no version older than 6.33 is used + public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, DeployLogger logger, Optional<Version> vespaVersion) { + this.appDirs = appDirs; + this.vespaVersion = vespaVersion; } - public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir) { - return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), new BaseDeployLogger(), Optional.<Version>empty()); + public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, Optional<Version> vespaVersion) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } - // Verify that files a and b does not coexist. - private void checkConflicts(String a, String b) throws IllegalArgumentException { - if (appDirs.file(a).exists() && appDirs.file(b).exists()) - throw new IllegalArgumentException("Application package in " + appDirs.root() + " contains both " + a + " and " + b + - ", please use just one of them"); + public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), Optional.<Version>empty()); } @SuppressWarnings("deprecation") public void checkApplication() throws IOException { validateHostsFile(SchemaValidator.hostsXmlSchemaName); validateServicesFile(SchemaValidator.servicesXmlSchemaName); + // TODO: Disable temporarily, need to get out feature to support ignoring validation errors + //validateDeploymentFile(SchemaValidator.deploymentXmlSchemaName); if (appDirs.searchdefinitions().exists()) { if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { @@ -85,7 +83,6 @@ public class ApplicationPackageXmlFilesValidator { if (appDirs.file(FilesApplicationPackage.HOSTS).exists()) { validate(hostsXmlSchemaName, FilesApplicationPackage.HOSTS); } - } private void validateServicesFile(String servicesXmlSchemaName) throws IOException { @@ -93,6 +90,12 @@ public class ApplicationPackageXmlFilesValidator { validate(servicesXmlSchemaName, servicesFileName()); } + private void validateDeploymentFile(String deploymentXmlSchemaName) throws IOException { + if (appDirs.file(FilesApplicationPackage.DEPLOYMENT_FILE.getName()).exists()) { + validate(deploymentXmlSchemaName, FilesApplicationPackage.DEPLOYMENT_FILE.getName()); + } + } + private void validate(String schemaName, String xmlFileName) throws IOException { createSchemaValidator(schemaName, vespaVersion).validate(appDirs.file(xmlFileName)); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 3b85e617f87..002c31d5910 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -44,10 +44,8 @@ import java.net.URL; import java.security.MessageDigest; import java.util.*; import java.util.jar.JarFile; -import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.io.IOUtils.readAll; import static com.yahoo.text.Lowercase.toLowerCase; @@ -626,13 +624,27 @@ public class FilesApplicationPackage implements ApplicationPackage { } @Override + public void validateXML() throws IOException { + validateXML(Optional.empty()); + } + + // TODO: Remove when no version older than 6.33 is used + @Override public void validateXML(DeployLogger logger) throws IOException { - validateXML(logger, Optional.empty()); + validateXML(Optional.empty()); + } + + @Override + public void validateXML(Optional<Version> vespaVersion) throws IOException { + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, vespaVersion); + xmlFilesValidator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); } + // TODO: Remove when no version older than 6.33 is used @Override public void validateXML(DeployLogger logger, Optional<Version> vespaVersion) throws IOException { - ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, logger, vespaVersion); + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, vespaVersion); xmlFilesValidator.checkApplication(); ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); } @@ -659,10 +671,10 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { IOUtils.recursiveDeleteDir(preprocessedDir); - IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> !name.equals(".preprocessed") && - !name.equals(SERVICES) && - !name.equals(HOSTS) && - !name.equals(CONFIG_DEFINITIONS_DIR)); + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(".preprocessed") && + ! name.equals(SERVICES) && + ! name.equals(HOSTS) && + ! name.equals(CONFIG_DEFINITIONS_DIR)); preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); if (getHostsFile().exists()) { preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index 334fda6e6eb..ce63ad23852 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -35,8 +35,4 @@ public class MockFileRegistry implements FileRegistry { return result; } - @Override - public Set<String> allRelativePaths() { - return Collections.emptySet(); - } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 67a24e0159b..ed4ccf51ff7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -83,11 +83,6 @@ public class PreGeneratedFileRegistry implements FileRegistry { } @Override - public Set<String> allRelativePaths() { - return path2Hash.keySet(); - } - - @Override public List<Entry> export() { List<Entry> entries = new ArrayList<>(); for (Map.Entry<String, String> entry : path2Hash.entrySet()) { diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java index a28a17dc831..698fa8fdce7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -43,6 +43,7 @@ public class SchemaValidator { public static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); static final String servicesXmlSchemaName = "services.rnc"; static final String hostsXmlSchemaName = "hosts.rnc"; + static final String deploymentXmlSchemaName = "deployment.rnc"; private final CustomErrorHandler errorHandler = new CustomErrorHandler(); private final ValidationDriver driver; private DeployLogger deployLogger; @@ -91,6 +92,15 @@ public class SchemaValidator { return new SchemaValidator(hostsXmlSchemaName); } + /** + * Create a validator for deployment.xml for tests + * + * @throws IOException if it is not possible to read schema files + */ + public static SchemaValidator createTestValidatorDeployment() throws IOException { + return new SchemaValidator(deploymentXmlSchemaName); + } + private class CustomErrorHandler implements ErrorHandler { volatile String fileName; diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java new file mode 100644 index 00000000000..338302e9e57 --- /dev/null +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import org.custommonkey.xmlunit.XMLUnit; +import org.junit.Test; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; +import javax.xml.transform.TransformerException; +import java.io.IOException; +import java.io.StringReader; + +/** + * @author bratseth + */ +public class HostedOverrideProcessorTest { + + static { + XMLUnit.setIgnoreWhitespace(true); + } + + private static final String input = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='1'/>" + + " <nodes deploy:environment=\"staging\" count='2'/>" + + " <nodes deploy:environment=\"prod\" count='3'/>" + + " <nodes deploy:environment=\"prod\" deploy:region=\"us-west\" count='4'/>" + + " </container>" + + "</services>"; + + + @Test + public void testParsingDefault() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.test, RegionName.defaultName(), expected); + } + + @Test + public void testParsingEnvironmentAndRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='4' required='true'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.from("prod"), RegionName.from("us-west"), expected); + } + + @Test + public void testParsingEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='3' required='true'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.valueOf("prod"), RegionName.from("us-east"), expected); + } + + @Test + public void testParsingEnvironmentNoRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='3' required='true'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.from("prod"), RegionName.defaultName(), expected); + } + + @Test + public void testParsingUnknownEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.from("dev"), RegionName.defaultName(), expected); + } + + @Test + public void testParsingUnknownEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.from("test"), RegionName.from("us-west"), expected); + } + + @Test + public void testParsingInheritEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException { + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + + " <container id=\"foo\" version=\"1.0\">" + + " <nodes count='2' required='true'/>" + + " </container>" + + "</services>"; + assertOverride(Environment.from("staging"), RegionName.from("us-west"), expected); + } + + private void assertOverride(Environment environment, RegionName region, String expected) throws TransformerException { + Document inputDoc = Xml.getDocument(new StringReader(input)); + Document newDoc = new OverrideProcessor(environment, region).process(inputDoc); + TestBase.assertDocument(expected, newDoc); + } + +} diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index 07068e236cd..6d9bf2cbfa5 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -17,6 +17,7 @@ import java.nio.file.NoSuchFileException; * @since 5.22 */ public class IncludeProcessorTest { + @Test public void testInclude() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { File app = new File("src/test/resources/multienvapp"); @@ -68,7 +69,7 @@ public class IncludeProcessorTest { "</jdisc></services>"; Document doc = (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app))); - System.out.println(Xml.documentAsString(doc)); + // System.out.println(Xml.documentAsString(doc)); TestBase.assertDocument(expected, doc); } @@ -78,4 +79,5 @@ public class IncludeProcessorTest { DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app))); } + } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java index eecbb1e7313..f6528e84368 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java @@ -83,7 +83,7 @@ public class XmlPreprocessorTest { "</services>"; Document docUsWest = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west"))).run(); - System.out.println(Xml.documentAsString(docUsWest)); + // System.out.println(Xml.documentAsString(docUsWest)); TestBase.assertDocument(expectedUsWest, docUsWest); String expectedUsEast = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + @@ -162,4 +162,5 @@ public class XmlPreprocessorTest { Document docDev = (new XmlPreProcessor(appDir, new StringReader(input), Environment.prod, RegionName.from("default")).run()); TestBase.assertDocument(expectedProd, docDev); } + } |