diff options
Diffstat (limited to 'config-application-package/src/main/java')
24 files changed, 2786 insertions, 0 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java b/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java new file mode 100644 index 00000000000..9a825662c4d --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.config.model.application.provider.Bundle; + +import java.io.*; +import java.util.List; + +/** + * A @{link ConfigDefinitionDir} contains a set of config definitions. New definitions may be added, + * but they cannot conflict with the existing ones. + * + * @author lulf + * @since 5.1 + */ +public class ConfigDefinitionDir { + private final File defDir; + + public ConfigDefinitionDir(File defDir) { + this.defDir = defDir; + } + + public void addConfigDefinitionsFromBundle(Bundle bundle, List<Bundle> bundlesAdded) { + try { + checkAndCopyUserDefs(bundle, bundlesAdded); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to add config definitions from bundle " + bundle.getFile().getAbsolutePath(), e); + } + } + + private void checkAndCopyUserDefs(Bundle bundle, List<Bundle> bundlesAdded) throws IOException { + for (Bundle.DefEntry def : bundle.getDefEntries()) { + checkUserDefConflict(bundle, def, bundlesAdded); + String defFilename = def.defNamespace + "." + def.defName + ".def"; + OutputStream out = new FileOutputStream(new File(defDir, defFilename)); + out.write(def.contents.getBytes()); + out.close(); + } + } + + private void checkUserDefConflict(Bundle bundle, Bundle.DefEntry userDef, List<Bundle> bundlesAdded) { + final String defName = userDef.defName; + final String defNamespace = userDef.defNamespace; + File[] builtinDefsWithSameName = defDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.matches(defName + ".def") || name.matches(defNamespace + "." + defName + ".def"); + } + }); + if (builtinDefsWithSameName != null && builtinDefsWithSameName.length > 0) { + String message = "a built-in config definition (" + getFilePathsCommaSeparated(builtinDefsWithSameName) + ")"; + for (Bundle b : bundlesAdded) { + for (Bundle.DefEntry defEntry : b.getDefEntries()) { + if (defEntry.defName.equals(defName) && defEntry.defNamespace.equals(defNamespace)) { + message = "the same config definition in the bundle '" + b.getFile().getName() + "'"; + } + } + } + throw new IllegalArgumentException("The config definition with name '" + defNamespace + "." + defName + + "' contained in the bundle '" + bundle.getFile().getName() + + "' conflicts with " + message + ". Please choose a different name."); + } + } + + private String getFilePathsCommaSeparated(File[] files) { + StringBuilder sb = new StringBuilder(); + if (files.length > 0) { + sb.append(files[0].getAbsolutePath()); + for (int i = 1; i < files.length; i++) { + sb.append(", "); + sb.append(files[i].getAbsolutePath()); + } + } + return sb.toString(); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java new file mode 100644 index 00000000000..dc8fd010e75 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.io.IOUtils; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.List; + +/** + * Handles preprocess:include statements and returns a Document which has all the include statements resolved + * + * @author musum + * @since 5.22 + */ +class IncludeProcessor implements PreProcessor { + + private final File application; + public IncludeProcessor(File application) { + this.application = application; + + } + + public Document process(Document input) throws IOException, TransformerException { + Document doc = Xml.copyDocument(input); + includeFile(application, doc.getDocumentElement()); + return doc; + } + + private static void includeFile(File currentFolder, Element currentElement) throws IOException { + NodeList list = currentElement.getElementsByTagNameNS(XmlPreProcessor.preprocessNamespaceUri, "include"); + while (list.getLength() > 0) { + Element elem = (Element) list.item(0); + Element parent = (Element) elem.getParentNode(); + String filename = elem.getAttribute("file"); + boolean required = elem.hasAttribute("required") ? Boolean.parseBoolean(elem.getAttribute("required")) : true; + File file = new File(currentFolder, filename); + + Document subFile = IncludeProcessor.parseIncludeFile(file, parent.getTagName(), required); + includeFile(file.getParentFile(), subFile.getDocumentElement()); + + //System.out.println("document before merging: " + documentAsString(doc)); + IncludeProcessor.mergeInto(parent, XML.getChildren(subFile.getDocumentElement())); + //System.out.println("document after merging: " + documentAsString(doc)); + parent.removeChild(elem); + //System.out.println("document after removing child: " + documentAsString(doc)); + list = currentElement.getElementsByTagNameNS(XmlPreProcessor.preprocessNamespaceUri, "include"); + } + } + + + private static void mergeInto(Element destination, List<Element> subElements) { + // System.out.println("merging " + subElements.size() + " elements into " + destination.getTagName()); + for (Element subElement : subElements) { + Node copiedNode = destination.getOwnerDocument().importNode(subElement, true); + destination.appendChild(copiedNode); + } + } + + private static Document parseIncludeFile(File file, String parentTagName, boolean required) throws IOException { + StringWriter w = new StringWriter(); + final String startTag = "<" + parentTagName + " " + XmlPreProcessor.deployNamespace + "='" + XmlPreProcessor.deployNamespaceUri + "' " + XmlPreProcessor.preprocessNamespace + "='" + XmlPreProcessor.preprocessNamespaceUri + "'>"; + w.append(startTag); + if (file.exists() || required) { + w.append(IOUtils.readFile(file)); + } + final String endTag = "</" + parentTagName + ">"; + w.append(endTag); + return XML.getDocument(new StringReader(w.toString())); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java new file mode 100644 index 00000000000..f3da285f524 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.data.access.simple.Value; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import javax.xml.transform.TransformerException; +import java.util.*; +import java.util.logging.Logger; + +/** + * Handles overrides in a XML document according to the rules defined for multi environment application packages. + * + * @author lulf + * @since 5.22 + */ +class OverrideProcessor implements PreProcessor { + private static final Logger log = Logger.getLogger(OverrideProcessor.class.getName()); + + private final Environment environment; + private final RegionName region; + private static final String ATTR_ID = "id"; + private static final String ATTR_ENV = "environment"; + private static final String ATTR_REG = "region"; + + public OverrideProcessor(Environment environment, RegionName region) { + this.environment = environment; + this.region = region; + } + + public Document process(Document input) throws TransformerException { + log.log(LogLevel.DEBUG, "Preprocessing overrides with " + environment + "." + region); + Document ret = Xml.copyDocument(input); + Element root = ret.getDocumentElement(); + applyOverrides(root, Context.empty()); + return ret; + } + + private void applyOverrides(Element parent, Context context) { + context = getParentContext(parent, context); + + Map<String, List<Element>> elementsByTagName = elementsByTagNameAndId(XML.getChildren(parent)); + + retainOverriddenElements(elementsByTagName); + + // For each tag name, prune overrides + for (Map.Entry<String, List<Element>> entry : elementsByTagName.entrySet()) { + pruneOverrides(parent, entry.getValue(), context); + } + + // Repeat for remaining children; + for (Element child : XML.getChildren(parent)) { + applyOverrides(child, context); + // Remove attributes + child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV); + child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_REG); + } + } + + private Context getParentContext(Element parent, Context context) { + Optional<Environment> environment = context.environment; + RegionName region = context.region; + if ( ! environment.isPresent()) { + environment = getEnvironment(parent); + } + if (region.isDefault()) { + region = getRegion(parent); + } + return Context.create(environment, region); + } + + /** + * Prune overrides from parent according to deploy override rules. + * + * @param parent Parent {@link Element} above children. + * @param children Children where one {@link Element} will remain as the overriding element + * @param context Current context with environment and region. + */ + private void pruneOverrides(Element parent, List<Element> children, Context context) { + checkConsistentInheritance(children, context); + pruneNonMatchingEnvironmentsAndRegions(parent, children); + retainMostSpecificEnvironmentAndRegion(parent, children, context); + } + + /** + * Ensures that environment and region does not change from something non-default to something else. + */ + private void checkConsistentInheritance(List<Element> children, Context context) { + for (Element child : children) { + Optional<Environment> env = getEnvironment(child); + RegionName reg = getRegion(child); + if (env.isPresent() && context.environment.isPresent() && !env.equals(context.environment)) { + throw new IllegalArgumentException("Environment in child (" + env.get() + ") differs from that inherited from parent (" + context.environment + ") at " + child); + } + if (!reg.isDefault() && !context.region.isDefault() && !reg.equals(context.region)) { + throw new IllegalArgumentException("Region in child (" + reg + ") differs from that inherited from parent (" + context.region + ") at " + child); + } + } + } + + /** + * Prune elements that are not matching our environment and region + */ + private void pruneNonMatchingEnvironmentsAndRegions(Element parent, List<Element> children) { + Iterator<Element> elemIt = children.iterator(); + while (elemIt.hasNext()) { + Element child = elemIt.next(); + Optional<Environment> env = getEnvironment(child); + RegionName reg = getRegion(child); + if ((env.isPresent() && !environment.equals(env.get())) || (!reg.isDefault() && !region.equals(reg))) { + parent.removeChild(child); + elemIt.remove(); + } + } + } + + /** + * Find the most specific element and remove all others. + */ + private void retainMostSpecificEnvironmentAndRegion(Element parent, List<Element> children, Context context) { + Element bestMatchElement = null; + int bestMatch = 0; + for (Element child : children) { + int currentMatch = 1; + Optional<Environment> elementEnvironment = hasEnvironment(child) ? getEnvironment(child) : context.environment; + RegionName elementRegion = hasRegion(child) ? getRegion(child) : context.region; + if (elementEnvironment.isPresent() && elementEnvironment.get().equals(environment)) + currentMatch++; + if ( ! elementRegion.isDefault() && elementRegion.equals(region)) + currentMatch++; + + if (currentMatch > bestMatch) { + bestMatchElement = child; + bestMatch = currentMatch; + } + } + + // Remove elements not specific + for (Element child : children) { + if (child != bestMatchElement) { + parent.removeChild(child); + } + } + } + + /** + * Retains all elements where at least one element is overridden. Removes non-overridden elements from map. + */ + private void retainOverriddenElements(Map<String, List<Element>> elementsByTagName) { + Iterator<Map.Entry<String, List<Element>>> it = elementsByTagName.entrySet().iterator(); + while (it.hasNext()) { + List<Element> elements = it.next().getValue(); + boolean hasOverrides = false; + for (Element element : elements) { + if (hasEnvironment(element) || hasRegion(element)) { + hasOverrides = true; + } + } + if (!hasOverrides) { + it.remove(); + } + } + } + + private boolean hasRegion(Element element) { + return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_REG); + } + + private boolean hasEnvironment(Element element) { + return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV); + } + + private Optional<Environment> getEnvironment(Element element) { + String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV); + if (env == null || env.isEmpty()) { + return Optional.empty(); + } + return Optional.of(Environment.from(env)); + } + + private RegionName getRegion(Element element) { + String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_REG); + if (reg == null || reg.isEmpty()) { + return RegionName.defaultName(); + } + return RegionName.from(reg); + } + + private Map<String, List<Element>> elementsByTagNameAndId(List<Element> children) { + Map<String, List<Element>> elementsByTagName = new LinkedHashMap<>(); + // Index by tag name + for (Element child : children) { + String key = child.getTagName(); + if (child.hasAttribute(ATTR_ID)) { + key += child.getAttribute(ATTR_ID); + } + if (!elementsByTagName.containsKey(key)) { + elementsByTagName.put(key, new ArrayList<>()); + } + elementsByTagName.get(key).add(child); + } + return elementsByTagName; + } + + /** + * Represents environment and region in a given context. + */ + private static final class Context { + + final Optional<Environment> environment; + + final RegionName region; + + private Context(Optional<Environment> environment, RegionName region) { + this.environment = environment; + this.region = region; + } + + static Context empty() { + return new Context(Optional.empty(), RegionName.defaultName()); + } + + public static Context create(Optional<Environment> environment, RegionName region) { + return new Context(environment, region); + } + + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java new file mode 100644 index 00000000000..55575f7498b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import org.w3c.dom.Document; + +import javax.xml.transform.TransformerException; +import java.io.IOException; + +/** + * Performs pre-processing of XML document and returns new document that has been processed. + * + * @author lulf + * @since 5.21 + */ +public interface PreProcessor { + public Document process(Document input) throws IOException, TransformerException; +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java new file mode 100644 index 00000000000..f8cb9819b61 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import org.w3c.dom.*; + +import javax.xml.transform.TransformerException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.logging.Logger; + +/** + * Handles getting properties from services.xml and replacing references to properties with their real values + * + * @author musum + * @since 5.22 + */ +class PropertiesProcessor implements PreProcessor { + private final static Logger log = Logger.getLogger(PropertiesProcessor.class.getName()); + private final LinkedHashMap<String, String> properties; + + PropertiesProcessor() { + properties = new LinkedHashMap<>(); + } + + public Document process(Document input) throws TransformerException { + Document doc = Xml.copyDocument(input); + final Document document = buildProperties(doc); + applyProperties(document.getDocumentElement()); + return document; + } + + private Document buildProperties(Document input) { + NodeList list = input.getElementsByTagNameNS(XmlPreProcessor.preprocessNamespaceUri, "properties"); + while (list.getLength() > 0) { + Element propertiesElement = (Element) list.item(0); + //System.out.println("prop=" + propertiesElement); + Element parent = (Element) propertiesElement.getParentNode(); + for (Node node : XML.getChildren(propertiesElement)) { + //System.out.println("Found " + node.getNodeName() + ", " + node.getTextContent()); + final String propertyName = node.getNodeName(); + if (properties.containsKey(propertyName)) { + log.log(LogLevel.WARNING, "Duplicate definition for property '" + propertyName + "' detected"); + } + properties.put(propertyName, node.getTextContent()); + } + parent.removeChild(propertiesElement); + list = input.getElementsByTagNameNS(XmlPreProcessor.preprocessNamespaceUri, "properties"); + } + return input; + } + + private void applyProperties(Element parent) { + // System.out.println("applying properties for " + parent.getNodeName()); + final NamedNodeMap attributes = parent.getAttributes(); + for (int i = 0; i < attributes.getLength(); i++) { + Node a = attributes.item(i); + if (hasProperty(a)) { + replaceAttributePropertyWithValue(a); + } + } + + if (XML.getChildren(parent).isEmpty() && parent.getTextContent() != null) { + if (hasPropertyInElement(parent)) { + replaceElementPropertyWithValue(parent); + } + } + + // Repeat for remaining children; + for (Element child : XML.getChildren(parent)) { + applyProperties(child); + } + } + + private void replaceAttributePropertyWithValue(Node a) { + String propertyValue = a.getNodeValue(); + String replacedPropertyValue = replaceValue(propertyValue); + a.setNodeValue(replacedPropertyValue); + } + + private String replaceValue(String propertyValue) { + /* Use a list with keys sorted by length (longest key first) + Needed for replacing values where you have overlapping keys */ + ArrayList<String> keys = new ArrayList<>(properties.keySet()); + Collections.sort(keys, Collections.reverseOrder(Comparator.comparing(String::length))); + + for (String key : keys) { + String value = properties.get(key); + // Try to find exact match first and since this is done with longest key + // first, the else branch will only happen when there cannot be an exact + // match, i.e. where you want to replace only parts of the attribute or node value + if (propertyValue.equals("${" + key + "}")) { + final String regex = "\\$\\{" + key + "\\}"; + return propertyValue.replaceAll(regex, value); + } else if (propertyValue.contains(key)) { + return propertyValue.replaceAll("\\$\\{" + key + "\\}", value); + } + } + throw new IllegalArgumentException("Unable to find property replace in " + propertyValue); + } + + private void replaceElementPropertyWithValue(Node a) { + String propertyValue = a.getTextContent(); + String replacedPropertyValue = replaceValue(propertyValue); + a.setTextContent(replacedPropertyValue); + } + + private static boolean hasProperty(Node node) { + return hasProperty(node.getNodeValue()); + } + + private static boolean hasPropertyInElement(Node node) { + return hasProperty(node.getTextContent()); + } + + private static boolean hasProperty(String s) { + return s.matches("^.*\\$\\{.+\\}.*$"); + } + + public LinkedHashMap<String, String> getProperties() { + return properties; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java new file mode 100644 index 00000000000..0a0216e80d9 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import edu.umd.cs.findbugs.annotations.NonNull; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import java.io.*; +import javax.xml.transform.dom.DOMResult; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * Utilities for XML. + * + * @author musum + */ +public class Xml { + private static final Logger log = Logger.getLogger(Xml.class.getPackage().toString()); + + // Access to this needs to be synchronized (as it is in getDocumentBuilder() below) + private static final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + static { + factory.setNamespaceAware(true); + factory.setXIncludeAware(false); + } + + public static Document getDocument(Reader reader) { + Document doc; + try { + doc = getDocumentBuilder().parse(new InputSource(reader)); + } catch (SAXException | IOException e) { + throw new IllegalArgumentException(e); + } + return doc; + } + + /** + * Creates a new XML document builder. + * + * @return A new DocumentBuilder instance, or null if we fail to get one. + */ + private static synchronized DocumentBuilder getDocumentBuilder() { + try { + return factory.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + log.log(LogLevel.WARNING, "No XML parser available - " + e); + return null; + } + } + + static DocumentBuilder getPreprocessDocumentBuilder() throws ParserConfigurationException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setXIncludeAware(false); + factory.setValidating(false); + return factory.newDocumentBuilder(); + } + + static File getServices(File app) { + return new File(app, "services.xml"); // TODO Do not hard-code + } + + static Document copyDocument(Document input) throws TransformerException { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + DOMSource source = new DOMSource(input); + DOMResult result = new DOMResult(); + transformer.transform(source, result); + return (Document) result.getNode(); + } + + static String documentAsString(Document document, boolean prettyPrint) throws TransformerException { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + if (prettyPrint) { + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + } else { + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + } + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(writer)); + return writer.toString(); + } + + static String documentAsString(Document document) throws TransformerException { + return documentAsString(document, false); + } + + /** + * Utility method to get an XML element from a reader + * + * @param reader the {@link Reader} to get an xml element from + */ + public static Element getElement(Reader reader) { + return XML.getDocument(reader).getDocumentElement(); + } + + /** + * @return The root element of each xml file under pathFromAppRoot/ in the app package + */ + @NonNull + public static List<Element> allElemsFromPath(ApplicationPackage app, String pathFromAppRoot) { + List<Element> ret = new ArrayList<>(); + List<NamedReader> files = null; + try { + files = app.getFiles(Path.fromString(pathFromAppRoot), ".xml", true); + for (NamedReader reader : files) + ret.add(getElement(reader)); + } finally { + NamedReader.closeAll(files); + } + return ret; + } + + /** + * Will get all sub-elements under parent named "name", just like XML.getChildren(). Then look under + * pathFromAppRoot/ in the app package for XML files, parse them and append elements of the same name. + * + * @param parent parent XML node + * @param name name of elements to merge + * @param app an {@link ApplicationPackage} + * @param pathFromAppRoot path from application root + * @return list of all sub-elements with given name + */ + public static List<Element> mergeElems(Element parent, String name, ApplicationPackage app, String pathFromAppRoot) { + List<Element> children = XML.getChildren(parent, name); + List<Element> allFromFiles = allElemsFromPath(app, pathFromAppRoot); + for (Element fromFile : allFromFiles) { + for (Element inThatFile : XML.getChildren(fromFile, name)) { + children.add(inThatFile); + } + } + return children; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java new file mode 100644 index 00000000000..7c4ddb812c4 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; + +/** + * A preprocessor for services.xml files that handles deploy:environment, deploy:region, preprocess:properties, preprocess:include + * and create a new Document which is based on the supplied environment and region + * + * @author musum + * @since 5.22 + */ +public class XmlPreProcessor { + final static String deployNamespace = "xmlns:deploy"; + final static String deployNamespaceUri = "vespa"; + final static String preprocessNamespace = "xmlns:preprocess"; + final static String preprocessNamespaceUri = "properties"; //TODO + + private final File applicationDir; + private final Reader xmlInput; + private final Environment environment; + private final RegionName region; + private final List<PreProcessor> chain; + + public XmlPreProcessor(File applicationDir, File xmlInput, Environment environment, RegionName region) throws IOException { + this(applicationDir, new FileReader(xmlInput), environment, region); + } + + public XmlPreProcessor(File applicationDir, Reader xmlInput, Environment environment, RegionName region) throws IOException { + this.applicationDir = applicationDir; + this.xmlInput = xmlInput; + this.environment = environment; + this.region = region; + this.chain = setupChain(); + } + + public Document run() throws ParserConfigurationException, IOException, SAXException, TransformerException { + DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); + final Document document = docBuilder.parse(new InputSource(xmlInput)); + return execute(document); + } + + private Document execute(Document input) throws IOException, TransformerException { + for (PreProcessor preProcessor : chain) { + input = preProcessor.process(input); + } + return input; + } + + private List<PreProcessor> setupChain() throws IOException { + List<PreProcessor> chain = new ArrayList<>(); + chain.add(new IncludeProcessor(applicationDir)); + chain.add(new OverrideProcessor(environment, region)); + chain.add(new PropertiesProcessor()); + return chain; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/application/package-info.java b/config-application-package/src/main/java/com/yahoo/config/application/package-info.java new file mode 100644 index 00000000000..d5c21b8549b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/application/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.application; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java b/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java new file mode 100644 index 00000000000..65b3b904c2b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.application; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java new file mode 100644 index 00000000000..6e730ba8410 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.io.File; + +/** + * Definitions of sub-directories of an application package. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class AppSubDirs { + + final Tuple2<File, String> root; + final Tuple2<File, String> templates; + public final Tuple2<File, String> rules; + final Tuple2<File, String> searchchains; + final Tuple2<File, String> docprocchains; + final Tuple2<File, String> routingtables; + final Tuple2<File, String> configDefs; + final Tuple2<File, String> searchdefinitions; + + public AppSubDirs(File root) { + this.root = new Tuple2<>(root, root.getName()); + templates = createTuple(ApplicationPackage.TEMPLATES_DIR); + rules = createTuple(ApplicationPackage.RULES_DIR.getRelative()); + searchchains = createTuple(ApplicationPackage.SEARCHCHAINS_DIR); + docprocchains = createTuple(ApplicationPackage.DOCPROCCHAINS_DIR); + routingtables = createTuple(ApplicationPackage.ROUTINGTABLES_DIR); + configDefs = createTuple(ApplicationPackage.CONFIG_DEFINITIONS_DIR); + searchdefinitions = createTuple(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + } + + private Tuple2<File, String> createTuple(String name) { + return new Tuple2<>(file(name), name); + } + + public File file(String subPath) { + return new File(root.first, subPath); + } + + public File root() { + return root.first; + } + + public File templates() { + return templates.first; + } + + public File rules() { + return rules.first; + } + + public File searchchains() { + return searchchains.first; + } + + public File docprocchains() { + return docprocchains.first; + } + + public File configDefs() { + return configDefs.first; + } + + public File searchdefinitions() { + return searchdefinitions.first; + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java new file mode 100644 index 00000000000..16c3ef3e029 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.Version; +import com.yahoo.path.Path; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +/** + * Validation of xml files in application package against RELAX NG schemas. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class ApplicationPackageXmlFilesValidator { + + private final AppSubDirs appDirs; + private final DeployLogger logger; + private final Optional<Version> vespaVersion; + + private static final FilenameFilter xmlFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".xml"); + } + }; + + public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, DeployLogger logger, Optional<Version> vespaVersion) { + this.appDirs = appDirs; + this.logger = logger; + this.vespaVersion = vespaVersion; + } + + public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, DeployLogger logger, Optional<Version> vespaVersion) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), logger, vespaVersion); + } + + public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), new BaseDeployLogger(), Optional.<Version>empty()); + } + + // Verify that files a and b does not coexist. + private void checkConflicts(String a, String b) throws IllegalArgumentException { + if (appDirs.file(a).exists() && appDirs.file(b).exists()) + throw new IllegalArgumentException("Application package in " + appDirs.root() + " contains both " + a + " and " + b + + ", please use just one of them"); + } + + @SuppressWarnings("deprecation") + public void checkApplication() throws IOException { + validateHostsFile(SchemaValidator.hostsXmlSchemaName); + validateServicesFile(SchemaValidator.servicesXmlSchemaName); + + if (appDirs.searchdefinitions().exists()) { + if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { + throw new IllegalArgumentException("Application package in " + appDirs.root() + + " must contain at least one search definition (.sd) file when directory searchdefinitions/ exists."); + } + } + + validate(appDirs.routingtables, "routing-standalone.rnc"); + } + + // For testing + public static void checkIncludedDirs(ApplicationPackage app) throws IOException { + for (String includedDir : app.getUserIncludeDirs()) { + List<NamedReader> includedFiles = app.getFiles(Path.fromString(includedDir), ".xml", true); + for (NamedReader file : includedFiles) { + createSchemaValidator("container-include.rnc", Optional.empty()).validate(file); + } + } + } + + @SuppressWarnings("deprecation") + private void validateHostsFile(String hostsXmlSchemaName) throws IOException { + if (appDirs.file(FilesApplicationPackage.HOSTS).exists()) { + validate(hostsXmlSchemaName, FilesApplicationPackage.HOSTS); + } + + } + + private void validateServicesFile(String servicesXmlSchemaName) throws IOException { + // vespa-services.xml or services.xml. Fallback to vespa-services.xml + validate(servicesXmlSchemaName, servicesFileName()); + } + + private void validate(String schemaName, String xmlFileName) throws IOException { + createSchemaValidator(schemaName, vespaVersion).validate(appDirs.file(xmlFileName)); + } + + @SuppressWarnings("deprecation") + private String servicesFileName() { + String servicesFile = FilesApplicationPackage.SERVICES; + if (!appDirs.file(servicesFile).exists()) { + throw new IllegalArgumentException("Application package in " + appDirs.root() + + " must contain " + FilesApplicationPackage.SERVICES); + } + return servicesFile; + } + + private void validate(Tuple2<File, String> directory, String schemaFile) throws IOException { + if ( ! directory.first.isDirectory()) return; + validate(directory, createSchemaValidator(schemaFile, vespaVersion)); + } + + private void validate(Tuple2<File, String> directory, SchemaValidator validator) throws IOException { + File dir = directory.first; + if ( ! dir.isDirectory()) return; + + String directoryName = directory.second; + for (File f : dir.listFiles(xmlFilter)) { + if (f.isDirectory()) + validate(new Tuple2<>(f, directoryName + File.separator + f.getName()),validator); + else + validator.validate(f, directoryName + File.separator + f.getName()); + } + } + + private static SchemaValidator createSchemaValidator(String schemaFile, Optional<Version> vespaVersion) { + return new SchemaValidator(schemaFile, new BaseDeployLogger(), vespaVersion); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java new file mode 100644 index 00000000000..50412348893 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.api.DeployLogger; + +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * Only logs to a normal {@link Logger} + * @author vegardh + * + */ +public final class BaseDeployLogger implements DeployLogger { + + private static final Logger log = Logger.getLogger("DeployLogger"); + + @Override + public final void log(Level level, String message) { + log.log(level, message); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java new file mode 100644 index 00000000000..8f2026afc66 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.google.common.base.Charsets; +import com.yahoo.collections.Tuple2; +import com.yahoo.config.codegen.CNode; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.*; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; + +/** + * A Bundle represents an OSGi bundle inside the model, and provides utilities + * for accessing resources within that bundle. + * + * @author tonytv, lulf + * @since 5.1 + */ +public class Bundle { + private static final Logger log = Logger.getLogger(Bundle.class.getName()); + private static final String DEFPATH = "configdefinitions/"; // path inside jar file + private final File bundleFile; + private final JarFile jarFile; + private final List<DefEntry> defEntries; + + public Bundle(JarFile jarFile, File bundleFile) { + this.jarFile = jarFile; + this.bundleFile = bundleFile; + defEntries = findDefEntries(); + } + + public static List<Bundle> getBundles(File bundleDir) { + try { + List<Bundle> bundles = new ArrayList<>(); + for (File bundleFile : getBundleFiles(bundleDir)) { + JarFile jarFile; + try { + jarFile = new JarFile(bundleFile); + } catch (ZipException e) { + throw new IllegalArgumentException("Error opening jar file '" + bundleFile.getName() + + "'. Please check that this is a valid jar file"); + } + bundles.add(new Bundle(jarFile, bundleFile)); + } + return bundles; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static List<File> getBundleFiles(File bundleDir) { + if (!bundleDir.isDirectory()) { + return new ArrayList<>(); + } + return Arrays.asList(bundleDir.listFiles( + new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + } + })); + } + + public List<DefEntry> getDefEntries() { + return Collections.unmodifiableList(defEntries); + } + + /** + * Returns a list of all .def-file entries in this Component. + * @return A list of .def-file entries. + */ + private List<DefEntry> findDefEntries() { + List<DefEntry> defEntries = new ArrayList<>(); + + ZipEntry defDir = jarFile.getEntry(DEFPATH); + + if ((defDir == null) || !defDir.isDirectory()) + return defEntries; + + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + + if (name.endsWith(".def")) { + if (name.matches("^" + DEFPATH + ".*\\.def$")) { + defEntries.add(new DefEntry(this, entry)); + } else { + log.info("Config definition file '" + name + "' in component '" + jarFile.getName() + + "' will not be used. Files must reside in the '" + DEFPATH + + "' directory in the .jar file"); + + } + } + } + return defEntries; + } + + public JarFile getJarFile() { + return jarFile; + } + + public File getFile() { + return bundleFile; + } + + /** + * Represents a def-file inside a Component. Immutable. + */ + public static class DefEntry { + + private final Bundle bundle; + private final ZipEntry zipEntry; + public final String defName; // Without version number and suffix. + public final String defNamespace; + public final String contents; + + /** + * @param bundle The bundle this def entry belongs to. + * @param zipEntry The ZipEntry representing the def-file. + */ + public DefEntry(Bundle bundle, ZipEntry zipEntry) { + this.bundle = bundle; + this.zipEntry = zipEntry; + + String entryName = zipEntry.getName(); + Tuple2<String, String> nameAndNamespace = ConfigUtils.getNameAndNamespaceFromString(entryName.substring(DEFPATH.length(), entryName.indexOf(".def"))); + + defName = nameAndNamespace.first; + defNamespace = getNamespace(); + if (defNamespace.isEmpty()) + throw new IllegalArgumentException("Config definition '" + defName + "' is missing a namespace"); + contents = getContents(); + } + + /** + * Returns the namespace of the .def-file, as given by the "namespace=" statement inside the given entry. + * @return The namespace string, or "" (empty string) if no namespace exists + */ + private String getNamespace() { + return ConfigUtils.getDefNamespace(getReader()); + } + + private String getContents() { + StringBuilder ret = new StringBuilder(""); + BufferedReader reader = new BufferedReader(getReader()); + try { + String str = reader.readLine(); + while (str != null){ + ret.append(str); + str = reader.readLine(); + if (str != null) { + ret.append("\n"); + } + } + reader.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Failed reading contents of def-file '" + defName + + ".def in component " + bundle.jarFile.getName(),e); + } + return ret.toString(); + } + + public Reader getReader() { + if (zipEntry == null) { + return new StringReader(""); + } + try { + return new InputStreamReader(bundle.jarFile.getInputStream(zipEntry), Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException("IOException", e); + } + } + + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java new file mode 100644 index 00000000000..25aacdfd74b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +/** + * A class for holding values generated or computed during deployment + * + * @author musum + * @since 5.1.11 + */ +public class DeployData { + /* Which user deployed */ + private final String deployedByUser; + + /* Name of application given by user */ + private final String applicationName; + + /* The absolute path to the directory holding the application */ + private final String deployedFromDir; + + /* Timestamp when a deployment was made */ + private final long deployTimestamp; + + /* Application generation. Incremented by one each time an application is deployed. */ + private final long generation; + private final long currentlyActiveGeneration; + + public DeployData(String deployedByUser, String deployedFromDir, String applicationName, Long deployTimestamp, Long generation, long currentlyActiveGeneration) { + this.deployedByUser = deployedByUser; + this.deployedFromDir = deployedFromDir; + this.applicationName = applicationName; + this.deployTimestamp = deployTimestamp; + this.generation = generation; + this.currentlyActiveGeneration = currentlyActiveGeneration; + } + + public String getDeployedByUser() { + return deployedByUser; + } + + public String getDeployedFromDir() { + return deployedFromDir; + } + + public long getDeployTimestamp() { + return deployTimestamp; + } + + public long getGeneration() { + return generation; + } + + public long getCurrentlyActiveGeneration() { + return currentlyActiveGeneration; + } + + public String getApplicationName() { + return applicationName; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java new file mode 100644 index 00000000000..735b8111f02 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; + +import java.lang.reflect.Constructor; + +/** + * Convenience for creating a {@link com.yahoo.config.FileReference}. + * + * @author gjoranv + */ +public class FileReferenceCreator { + + public static FileReference create(String stringVal) { + try { + Constructor<FileReference> ctor = FileReference.class.getDeclaredConstructor(String.class); + ctor.setAccessible(true); + return ctor.newInstance(stringVal); + } catch (Exception e) { + throw new RuntimeException("Could not create a new " + FileReference.class.getName() + + ". This should never happen!", e); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java new file mode 100644 index 00000000000..67a180d334d --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author lulf + * @author vegardh + * @since 5.1 + */ +public class FilesApplicationFile extends ApplicationFile { + private static final Logger log = Logger.getLogger("FilesApplicationFile"); + private final File file; + private final ObjectMapper mapper = new ObjectMapper(); + public FilesApplicationFile(Path path, File file) { + super(path); + this.file = file; + } + + @Override + public boolean isDirectory() { + return file.isDirectory(); + } + + @Override + public boolean exists() { + return file.exists(); + } + + @Override + public ApplicationFile delete() { + log.log(LogLevel.DEBUG, "Delete " + file); + if (file.isDirectory() && !listFiles().isEmpty()) { + throw new RuntimeException("files. Can't delete, directory not empty: " + this + "(" + listFiles() + ")." + listFiles().size()); + } + if (file.isDirectory() && file.listFiles() != null && file.listFiles().length > 0) { + for (File f : file.listFiles()) { + deleteFile(f); + } + } + if (!file.delete()) { + throw new IllegalStateException("Unable to delete: "+this); + } + try { + writeMetaFile("", ApplicationFile.ContentStatusDeleted); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + public static boolean deleteFile(File path) { + if( path.exists() ) { + if (path.isDirectory()) { + File[] files = path.listFiles(); + for(int i=0; i<files.length; i++) { + if(files[i].isDirectory()) { + deleteFile(files[i]); + } else { + files[i].delete(); + } + } + } + } + return(path.delete()); + } + + @Override + public Reader createReader() throws FileNotFoundException { + return new FileReader(file); + } + + @Override + public InputStream createInputStream() throws FileNotFoundException { + return new FileInputStream(file); + } + + @Override + public ApplicationFile createDirectory() { + if (file.isDirectory()) return this; + if (file.exists()) { + throw new IllegalArgumentException("Unable to create directory, file exists: "+file); + } + if (!file.mkdirs()) { + throw new IllegalArgumentException("Unable to create directory: "+file); + } + try { + writeMetaFile("", ApplicationFile.ContentStatusNew); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public ApplicationFile writeFile(Reader input) { + if (file.getParentFile() != null) { + file.getParentFile().mkdirs(); + } + try { + String data = com.yahoo.io.IOUtils.readAll(input); + String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew; + IOUtils.writeFile(file, data, false); + writeMetaFile(data, status); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public List<ApplicationFile> listFiles(final PathFilter filter) { + List<ApplicationFile> files = new ArrayList<>(); + if (!file.isDirectory()) { + return files; + } + FileFilter fileFilter = pathname -> filter.accept(path.append(pathname.getName())); + for (File child : file.listFiles(fileFilter)) { + // Ignore dot-files. + if (!child.getName().startsWith(".")) { + files.add(new FilesApplicationFile(path.append(child.getName()), child)); + } + } + return files; + } + + private void writeMetaFile(String data, String status) throws IOException { + File metaDir = createMetaDir(); + log.log(LogLevel.DEBUG, "meta dir=" + metaDir); + File metaFile = new File(metaDir + "/" + getPath().getName()); + if (status == null) { + status = ApplicationFile.ContentStatusNew; + if (metaFile.exists()) { + status = ApplicationFile.ContentStatusChanged; + } + } + String hash; + if (file.isDirectory() || status.equals(ApplicationFile.ContentStatusDeleted)) { + hash = ""; + } else { + hash = ConfigUtils.getMd5(data); + } + mapper.writeValue(metaFile, new MetaData(status, hash)); + } + + private File createMetaDir() { + File metaDir = getMetaDir(); + if (!metaDir.exists()) { + log.log(LogLevel.DEBUG, "Creating meta dir " + metaDir); + metaDir.mkdirs(); + } + return metaDir; + } + + private File getMetaDir() { + String substring = file.getAbsolutePath().substring(0, file.getAbsolutePath().lastIndexOf("/") + 1); + return new File(substring + Path.fromString(".meta/")); + } + + public MetaData getMetaData() { + File metaDir = getMetaDir(); + File metaFile = new File(metaDir + "/" + getPath().getName()); + log.log(LogLevel.DEBUG, "Getting metadata for " + metaFile); + if (metaFile.exists()) { + try { + return mapper.readValue(metaFile, MetaData.class); + } catch (IOException e) { + System.out.println("whot:" + Exceptions.toMessageString(e)); + // return below + } + } + try { + if (file.isDirectory()) { + return new MetaData(ApplicationFile.ContentStatusNew, ""); + } else { + return new MetaData(ApplicationFile.ContentStatusNew, ConfigUtils.getMd5(IOUtils.readAll(createReader()))); + } + } catch (IOException | IllegalArgumentException e) { + return null; + } + } + + @Override + public int compareTo(ApplicationFile other) { + if (other == this) return 0; + return this.getPath().getName().compareTo((other).getPath().getName()); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java new file mode 100644 index 00000000000..21e7de8fa7b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -0,0 +1,772 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.ConfigDefinitionDir; +import com.yahoo.config.application.Xml; +import com.yahoo.config.application.XmlPreProcessor; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.RuleConfigDeriver; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.path.Path; +import com.yahoo.io.HexDump; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.util.ConfigUtils; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.*; +import java.net.JarURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.security.MessageDigest; +import java.util.*; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.io.IOUtils.readAll; +import static com.yahoo.text.Lowercase.toLowerCase; + + +/** + * Application package derived from local files, ie. during deploy. + * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or + * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}. + * + * @author vegardh + */ +public class FilesApplicationPackage implements ApplicationPackage { + + private static final Logger log = Logger.getLogger(FilesApplicationPackage.class.getName()); + private static final String META_FILE_NAME = ".applicationMetaData"; + + private final File appDir; + private final File preprocessedDir; + private final File configDefsDir; + private final AppSubDirs appSubDirs; + // NOTE: these directories exist in the original user app, but their locations are given in 'services.xml' + private final List<String> userIncludeDirs = new ArrayList<>(); + private final ApplicationMetaData metaData; + + /** + * Returns an application package object based on the given application dir + * + * @param appDir application package directory + * @return an Application package instance + */ + public static FilesApplicationPackage fromFile(File appDir) { + return new Builder(appDir).preprocessedDir(new File(appDir, ".preprocessed")).build(); + } + + /** Creates package from a local directory, typically deploy app */ + public static FilesApplicationPackage fromFileWithDeployData(File appDir, DeployData deployData) { + return new Builder(appDir).deployData(deployData).build(); + } + + /** + * Builder for {@link com.yahoo.config.model.application.provider.FilesApplicationPackage}. Use + * this to create instances in a flexible manner. + */ + public static class Builder { + private final File appDir; + private Optional<File> preprocessedDir = Optional.empty(); + private Optional<ApplicationMetaData> metaData = Optional.empty(); + + public Builder(File appDir) { + this.appDir = appDir; + } + + public Builder preprocessedDir(File preprocessedDir) { + this.preprocessedDir = Optional.ofNullable(preprocessedDir); + return this; + } + + public Builder deployData(DeployData deployData) { + this.metaData = Optional.of(metaDataFromDeployData(appDir, deployData)); + return this; + } + + public FilesApplicationPackage build() { + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, ".preprocessed")), metaData.orElse(readMetaData(appDir))); + } + } + + private static ApplicationMetaData metaDataFromDeployData(File appDir, DeployData deployData) { + return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), deployData.getDeployTimestamp(), + deployData.getApplicationName(), computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); + } + + /** + * New package from given path on local file system. Retrieves config definition files from + * the default location 'serverdb/classes'. + * + * @param appDir application package directory + * @param preprocessedDir preprocessed application package output directory + * @param metaData metadata for this application package + */ + @SuppressWarnings("deprecation") + private FilesApplicationPackage(File appDir, File preprocessedDir, ApplicationMetaData metaData) { + verifyAppDir(appDir); + this.appDir = appDir; + this.preprocessedDir = preprocessedDir; + appSubDirs = new AppSubDirs(appDir); + configDefsDir = new File(appDir, ApplicationPackage.CONFIG_DEFINITIONS_DIR); + addUserIncludeDirs(); + this.metaData = metaData; + } + + public String getApplicationName() { + return metaData.getApplicationName(); + } + + @Override + public List<NamedReader> getFiles(Path relativePath, String suffix, boolean recurse) { + return getFiles(relativePath, "", suffix, recurse); + } + + @Override + public ApplicationFile getFile(Path path) { + File file = (path.isRoot() ? appDir : new File(appDir, path.getRelative())); + return new FilesApplicationFile(path, file); + } + + @Override + public ApplicationMetaData getMetaData() { + return metaData; + } + + private List<NamedReader> getFiles(Path relativePath,String namePrefix,String suffix,boolean recurse) { + try { + List<NamedReader> readers=new ArrayList<>(); + File dir = new File(appDir, relativePath.getRelative()); + if ( ! dir.isDirectory()) return readers; + + final File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + if (recurse) + readers.addAll(getFiles(relativePath.append(file.getName()), namePrefix + "/" + file.getName(), suffix, recurse)); + } else { + if (suffix == null || file.getName().endsWith(suffix)) + readers.add(new NamedReader(file.getName(), new FileReader(file))); + } + } + } + return readers; + } + catch (IOException e) { + throw new RuntimeException("Could not open (all) files in '" + relativePath + "'",e); + } + } + + private void verifyAppDir(File appDir) { + if (appDir==null || !appDir.isDirectory()) { + throw new IllegalArgumentException("Path '" + appDir + "' is not a directory."); + } + if (! appDir.canRead()){ + throw new IllegalArgumentException("Cannot read from application directory '" + appDir + "'"); + } + } + + @Override + public Reader getHosts() { + try { + File hostsFile = getHostsFile(); + if (!hostsFile.exists()) return null; + return new FileReader(hostsFile); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public String getHostSource() { + return getHostsFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getHostsFile() { + return new File(appDir, ApplicationPackage.HOSTS); + } + + private File getFileWithFallback(String first, String second) { + File firstFile = new File(appDir, first); + File secondFile = new File(appDir, second); + if (firstFile.exists()) { + return firstFile; + } else if (secondFile.exists()) { + return secondFile; + } else { + return firstFile; + } + } + + @Override + public String getServicesSource() { + return getServicesFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getServicesFile() { + return new File(appDir, ApplicationPackage.SERVICES); + } + + @Override + public Optional<Reader> getDeployment() { return optionalFile(DEPLOYMENT_FILE); } + + @Override + public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES); } + + private Optional<Reader> optionalFile(Path filePath) { + try { + return Optional.of(getFile(filePath).createReader()); + } catch (FileNotFoundException e) { + return Optional.empty(); + } + } + + + @Override + public List<String> getUserIncludeDirs() { + return Collections.unmodifiableList(userIncludeDirs); + } + + public void addUserIncludeDirs() { + Document services; + try { + services = Xml.getDocument(getServices()); + } catch (Exception e) { + return; // This method does not validate that services.xml exists, or that it is valid xml. + } + NodeList includeNodes = services.getElementsByTagName(IncludeDirs.INCLUDE); + + for (int i=0; i < includeNodes.getLength(); i++) { + Node includeNode = includeNodes.item(i); + addIncludeDir(includeNode); + } + } + + private void addIncludeDir(Node includeNode) { + if (! (includeNode instanceof Element)) + return; + Element include = (Element) includeNode; + if (! include.hasAttribute(IncludeDirs.DIR)) + return; + String dir = include.getAttribute(IncludeDirs.DIR); + validateIncludeDir(dir); + IncludeDirs.validateFilesInIncludedDir(dir, include.getParentNode(), this); + log.log(LogLevel.INFO, "Adding user include dir '" + dir + "'"); + userIncludeDirs.add(dir); + } + + @Override + public void validateIncludeDir(String dirName) { + IncludeDirs.validateIncludeDir(dirName, this); + } + + @Override + public Collection<NamedReader> searchDefinitionContents() { + Map<String, NamedReader> ret = new LinkedHashMap<>(); + Set<String> fileSds = new LinkedHashSet<>(); + Set<String> bundleSds = new LinkedHashSet<>(); + try { + for (File f : getSearchDefinitionFiles()) { + fileSds.add(f.getName()); + ret.put(f.getName(), new NamedReader(f.getName(), new FileReader(f))); + } + for (Map.Entry<String, String> e : allSdsFromDocprocBundlesAndClasspath(appDir).entrySet()) { + bundleSds.add(e.getKey()); + ret.put(e.getKey(), new NamedReader(e.getKey(), new StringReader(e.getValue()))); + } + } catch (Exception e) { + throw new IllegalArgumentException("Couldn't get search definition contents.", e); + } + verifySdsDisjoint(fileSds, bundleSds); + return ret.values(); + } + + /** + * Verify that two sets of search definitions are disjoint (TODO: everything except error message is very generic). + * @param fileSds Set of search definitions from file + * @param bundleSds Set of search definitions from bundles + */ + private void verifySdsDisjoint(Set<String> fileSds, Set<String> bundleSds) { + if (!Collections.disjoint(fileSds, bundleSds)) { + Collection<String> disjoint = new ArrayList<>(fileSds); + disjoint.retainAll(bundleSds); + throw new IllegalArgumentException("For the following search definitions names there are collisions between those specified inside " + + "docproc bundles and those in searchdefinitions/ in application package: "+disjoint); + } + } + + /** + * Returns sdName→payload for all SDs in all docproc bundles and on local classpath. + * Throws {@link IllegalArgumentException} if there are multiple sd files of same name. + * @param appDir application package directory + * @return a map from search definition name to search definition content + * @throws IOException if reading a search definition fails + */ + public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { + File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); + if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); + List<String> usedNames = new ArrayList<>(); + Map<String, String> ret = new LinkedHashMap<>(); + + // try classpath first + allSdsOnClassPath(usedNames, ret); + + for (File bundle : dpChains.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + }})) { + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+ + bundle.getName()+"' is already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + return ret; + } + + private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException { + Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + + while(resources.hasMoreElements()) { + URL resource = resources.nextElement(); + + String protocol = resource.getProtocol(); + + if ("file".equals(protocol)) { + File file; + try { + file = new File(resource.toURI()); + } catch (URISyntaxException e) { + continue; + } + // only interested in directories + if (file.isDirectory()) { + List<File> sdFiles = getSearchDefinitionFiles(file); + for (File sdFile : sdFiles) { + String sdName = sdFile.getName(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+ + "' found in classpath already used earlier in classpath."); + } + usedNames.add(sdName); + String contents = IOUtils.readAll(new FileReader(sdFile)); + ret.put(sdFile.getName(), contents); + } + } + } + else if ("jar".equals(protocol)) { + JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definitions name '"+sdName+ + "' used in bundle '"+jarFile.getName()+"' already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + } + } + + private Reader retrieveConfigDefReader(String defName) { + File def = new File(configDefsDir + File.separator + defName); + try { + return new NamedReader(def.getAbsolutePath(), new FileReader(def)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read config definition file '" + + def.getAbsolutePath() + "'", e); + } + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs = new LinkedHashMap<>(); + + if (configDefsDir.isDirectory()) { + addAllDefsFromConfigDir(defs, configDefsDir); + } + addAllDefsFromBundles(defs, FilesApplicationPackage.getComponents(appDir)); + return defs; + } + + private void addAllDefsFromBundles(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, List<Component> components) { + for (Component component : components) { + Bundle bundle = component.getBundle(); + for (final Bundle.DefEntry def : bundle.getDefEntries()) { + final ConfigDefinitionKey defKey = new ConfigDefinitionKey(def.defName, def.defNamespace); + if (!defs.containsKey(defKey)) { + defs.put(defKey, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(defKey.getName(), new StringReader(def.contents)); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return def.contents; + } + }); + } + } + } + } + + private void addAllDefsFromConfigDir(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, File configDefsDir) { + log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); + for (final File def : configDefsDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\.def");}})) { + + log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'"); + String[] nv = def.getName().split("\\.def"); + if (nv == null) { + log.log(LogLevel.WARNING, "Skipping '" + def + "', cannot determine name"); + } else { + ConfigDefinitionKey key; + try { + key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); + } catch (IOException e) { + e.printStackTrace(); + break; + } + if (key.getNamespace().isEmpty()) { + throw new IllegalArgumentException("Config definition '" + nv + "' has no namespace"); + } + boolean addFile = false; + if (defs.containsKey(key)) { + if (nv[0].contains(".")) { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". The file '" + def + "' will take precedence"); + addFile = true; + } else { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". Skipping '" + def + "', as it does not contain namespace in filename"); + } + } else { + addFile = true; + } + if (addFile) { + log.log(LogLevel.DEBUG, "Adding " + key + " to archive of all existing config defs"); + final ConfigDefinitionKey finalKey = key; + defs.put(key, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(finalKey.getName(), retrieveConfigDefReader(def.getName())); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return readConfigDefinition(def.getName()); + } + }); + } + } + } + } + + private String readConfigDefinition(String name) { + try (Reader reader = retrieveConfigDefReader(name)) { + return IOUtils.readAll(reader); + } catch (IOException e) { + throw new RuntimeException("Error reading config definition " + name, e); + } + } + + @Override + public Reader getServices() { + try { + return new FileReader(getServicesSource()); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + //Only intended for DeployProcessor, others should use the member version + static List<File> getSearchDefinitionFiles(File appDir) { + //The dot is escaped later in this method: + assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.'); + + List<File> ret = new ArrayList<>(); + File sdDir; + + sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + if (!sdDir.isDirectory()) { + return ret; + } + ret.addAll(Arrays.asList( + sdDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}}))); + return ret; + } + + public List<File> getSearchDefinitionFiles() { + return getSearchDefinitionFiles(appDir); + } + + //Only for use by deploy processor + public static List<Component> getComponents(File appDir) { + List<Component> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new Component(bundle, new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath()))); + } + return components; + } + + private static List<ComponentInfo> getComponentsInfo(File appDir) { + List<ComponentInfo> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath())); + } + return components; + } + + @Override + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + return getComponentsInfo(appDir); + } + + /** + * Returns a list of all components in this package. + * + * @return A list of components. + */ + public List<Component> getComponents() { + return getComponents(appDir); + } + + public File getAppDir() throws IOException { + return appDir.getCanonicalFile(); + } + + public static ApplicationMetaData readMetaData(File appDir) { + ApplicationMetaData defaultMetaData = new ApplicationMetaData(appDir, "n/a", "n/a", 0l, "", 0l, 0l); + File metaFile = new File(appDir, META_FILE_NAME); + if (!metaFile.exists()) { + return defaultMetaData; + } + try (FileReader reader = new FileReader(metaFile)) { + return ApplicationMetaData.fromJsonString(IOUtils.readAll(reader)); + } catch (Exception e) { + // Not a big deal, return default + return defaultMetaData; + } + } + + /** + * Represents a component in the application package. Immutable. + */ + public static class Component { + + public final ComponentInfo info; + private final Bundle bundle; + + public Component(Bundle bundle, ComponentInfo info) { + this.bundle = bundle; + this.info = info; + } + + public List<Bundle.DefEntry> getDefEntries() { + return bundle.getDefEntries(); + } + + public Bundle getBundle() { + return bundle; + } + } // class Component + + /** + * Reads a ranking expression from file to a string and returns it. + * + * @param name the name of the file to return, either absolute or + * relative to the search definition directory in the application package + * @return the content of a ranking expression file + * @throws IllegalArgumentException if the file was not found or could not be read + */ + // TODO: A note on absolute paths: We don't want to support this and it should be removed on 6.0 + // Currently one system test (basicmlr) depends on it. + @Override + public Reader getRankingExpression(String name) { + try { + return IOUtils.createReader(expressionFileNameToFile(name), "utf-8"); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read ranking expression file '" + name + "'", e); + } + } + + private File expressionFileNameToFile(String name) { + File expressionFile = new File(name); + if (expressionFile.isAbsolute()) return expressionFile; + + File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + return new File(sdDir, name); + } + + @Override + public File getFileReference(Path pathRelativeToAppDir) { + return new File(appDir, pathRelativeToAppDir.getRelative()); + } + + @Override + public void validateXML(DeployLogger logger) throws IOException { + validateXML(logger, Optional.empty()); + } + + @Override + public void validateXML(DeployLogger logger, Optional<Version> vespaVersion) throws IOException { + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, logger, vespaVersion); + xmlFilesValidator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); + } + + @Override + public void writeMetaData() throws IOException { + File metaFile = new File(appDir, META_FILE_NAME); + IOUtils.writeFile(metaFile, metaData.asJsonString(), false); + } + + @Override + public Collection<NamedReader> getSearchDefinitions() { + return searchDefinitionContents(); + } + + private void preprocessXML(File destination, File inputXml, Zone zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { + Document document = new XmlPreProcessor(appDir, inputXml, zone.environment(), zone.region()).run(); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + try (FileOutputStream outputStream = new FileOutputStream(destination)) { + transformer.transform(new DOMSource(document), new StreamResult(outputStream)); + } + } + + @Override + public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { + IOUtils.recursiveDeleteDir(preprocessedDir); + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> !name.equals(".preprocessed") && + !name.equals(SERVICES) && + !name.equals(HOSTS) && + !name.equals(CONFIG_DEFINITIONS_DIR)); + preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); + if (getHostsFile().exists()) { + preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); + } + FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir); + preprocessed.copyUserDefsIntoApplication(); + return preprocessed; + } + + private void copyUserDefsIntoApplication() { + File destination = appSubDirs.configDefs(); + destination.mkdir(); + ConfigDefinitionDir defDir = new ConfigDefinitionDir(destination); + // Copy the user's def files from components. + List<Bundle> bundlesAdded = new ArrayList<>(); + for (FilesApplicationPackage.Component component : FilesApplicationPackage.getComponents(appSubDirs.root())) { + Bundle bundle = component.getBundle(); + defDir.addConfigDefinitionsFromBundle(bundle, bundlesAdded); + bundlesAdded.add(bundle); + } + } + + /** + * Computes an md5 hash of the contents of the application package + * + * @return an md5sum of the application package + */ + private static String computeCheckSum(File appDir) { + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + for (File file : appDir.listFiles((dir, name) -> !name.equals(ApplicationPackage.EXT_DIR) && !name.startsWith("."))) { + addPathToDigest(file, "", md5, true, false); + } + return toLowerCase(HexDump.toHexString(md5.digest())); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Adds the given path to the digest, or does nothing if path is neither file nor dir + * @param path path to add to message digest + * @param suffix only files with this suffix are considered + * @param digest the {link @MessageDigest} to add the file paths to + * @param recursive whether to recursively find children in the paths + * @param fullPathNames Whether to include the full paths in checksum or only the names + * @throws java.io.IOException if adding path to digest fails when reading files from path + */ + private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException { + if (!path.exists()) return; + if (fullPathNames) { + digest.update(path.getPath().getBytes(Utf8.getCharset())); + } else { + digest.update(path.getName().getBytes(Utf8.getCharset())); + } + if (path.isFile()) { + FileInputStream is = new FileInputStream(path); + addToDigest(is, digest); + is.close(); + } else if (path.isDirectory()) { + final File[] files = path.listFiles(); + if (files != null) { + for (File elem : files) { + if ((elem.isDirectory() && recursive) || elem.getName().endsWith(suffix)) { + addPathToDigest(elem, suffix, digest, recursive, fullPathNames); + } + } + } + } + } + + private static final int MD5_BUFFER_SIZE = 65536; + private static void addToDigest(InputStream is, MessageDigest digest) throws IOException { + if (is==null) return; + byte[] buffer = new byte[MD5_BUFFER_SIZE]; + int i; + do { + i=is.read(buffer); + if (i > 0) { + digest.update(buffer, 0, i); + } + } while(i!=-1); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java new file mode 100644 index 00000000000..8d0707f9e99 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.io.File; +import java.util.List; + +/** + * Helper methods for directories included from services.xml in a <include dir=''/> element. + * + * @author gjoranv + * @since 5.1.19 + */ +public class IncludeDirs { + + public static final String INCLUDE = "include"; + public static final String DIR = "dir"; + + private IncludeDirs() { + throw new UnsupportedOperationException(IncludeDirs.class.getName() + " cannot be instantiated!"); + } + + public static void validateIncludeDir(String dirName, FilesApplicationPackage app) { + File file = new File(dirName); + + if (file.isAbsolute()) { + throw new IllegalArgumentException("Cannot include directory '" + dirName + + "', absolute paths are not supported. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + + file = app.getFileReference(Path.fromString(dirName)); + + if (!file.exists()) { + throw new IllegalArgumentException("Cannot include directory '" + dirName + + "', as it does not exist. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + + if (!file.isDirectory()) { + throw new IllegalArgumentException("Cannot include '" + dirName + + "', as it is not a directory. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + } + + + public static void validateFilesInIncludedDir(String dirName, Node parentNode, ApplicationPackage app) { + if (! (parentNode instanceof Element)) { + throw new IllegalStateException("The parent xml node of an include is not an Element: " + parentNode); + } + String parentTagName = ((Element) parentNode).getTagName(); + + List<Element> includedRootElems = Xml.allElemsFromPath(app, dirName); + for (Element includedRootElem : includedRootElems) { + validateIncludedFile(includedRootElem, parentTagName, dirName); + } + } + + /** + * @param includedRootElem The root element of the included file + * @param dirName The name of the included dir + */ + private static void validateIncludedFile(Element includedRootElem, String parentTagName, String dirName) { + if (!parentTagName.equals(includedRootElem.getTagName())) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' does not have <" + parentTagName + "> as root element."); + } + if (includedRootElem.hasAttributes()) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' has attributes set on its root element <" + parentTagName + + ">. These must be set in services.xml instead."); + } + if (XML.getChild(includedRootElem, INCLUDE) != null) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' has <include> subelement. Recursive inclusion is not supported."); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java new file mode 100644 index 00000000000..334fda6e6eb --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * A file registry for testing, and, it seems, doubling as a null registry in some code paths. + * + * @author tonytv + */ +public class MockFileRegistry implements FileRegistry { + + public FileReference addFile(String relativePath) { + return FileReferenceCreator.create("0123456789abcdef"); + } + + @Override + public String fileSourceHost() { + return "localhost.fortestingpurposesonly"; + } + + public static final Entry entry1 = new Entry("component/path1", FileReferenceCreator.create("1234")); + public static final Entry entry2 = new Entry("component/path2", FileReferenceCreator.create("56789")); + + public List<Entry> export() { + List<Entry> result = new ArrayList<>(); + result.add(entry1); + result.add(entry2); + return result; + } + + @Override + public Set<String> allRelativePaths() { + return Collections.emptySet(); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java new file mode 100644 index 00000000000..67a24e0159b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.*; +import java.util.regex.Pattern; + +/** + * Registry of files added earlier (i.e. during deployment) + * + * @author tonytv + */ +public class PreGeneratedFileRegistry implements FileRegistry { + + private final String fileSourceHost; + private final Map<String, String> path2Hash = new LinkedHashMap<>(); + + private static String entryDelimiter = "\t"; + private static Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); + + private PreGeneratedFileRegistry(Reader readerArg) { + BufferedReader reader = new BufferedReader(readerArg); + try { + fileSourceHost = reader.readLine(); + if (fileSourceHost == null) + throw new RuntimeException("Error while reading pre generated file registry"); + + String line; + while ((line = reader.readLine()) != null) { + addFromLine(line); + } + } catch(IOException e) { + throw new RuntimeException("Error while reading pre generated file registry", e); + } finally { + try { + reader.close(); + } catch(IOException e) {} + } + } + + private void addFromLine(String line) { + String[] parts = entryDelimiterPattern.split(line); + addEntry(parts[0], parts[1]); + } + + private void addEntry(String relativePath, String hash) { + path2Hash.put(relativePath, hash); + } + + public static String exportRegistry(FileRegistry registry) { + List<FileRegistry.Entry> entries = registry.export(); + StringBuilder builder = new StringBuilder(); + + builder.append(registry.fileSourceHost()).append('\n'); + for (FileRegistry.Entry entry : entries) { + builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()). + append('\n'); + } + + return builder.toString(); + } + + public static PreGeneratedFileRegistry importRegistry(Reader reader) { + return new PreGeneratedFileRegistry(reader); + } + + public FileReference addFile(String relativePath) { + return FileReferenceCreator.create(path2Hash.get(relativePath)); + } + + @Override + public String fileSourceHost() { + return fileSourceHost; + } + + public Set<String> getPaths() { + return path2Hash.keySet(); + } + + @Override + public Set<String> allRelativePaths() { + return path2Hash.keySet(); + } + + @Override + public List<Entry> export() { + List<Entry> entries = new ArrayList<>(); + for (Map.Entry<String, String> entry : path2Hash.entrySet()) { + entries.add(new Entry(entry.getKey(), FileReferenceCreator.create(entry.getValue()))); + } + return entries; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java new file mode 100644 index 00000000000..a28a17dc831 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.thaiopensource.util.PropertyMap; +import com.thaiopensource.util.PropertyMapBuilder; +import com.thaiopensource.validate.ValidateProperty; +import com.thaiopensource.validate.ValidationDriver; +import com.thaiopensource.validate.rng.CompactSchemaReader; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; +import org.xml.sax.ErrorHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.Enumeration; +import java.util.Optional; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Validates xml files against one schema. + * + * @author tonytv + */ +public class SchemaValidator { + + public static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); + static final String servicesXmlSchemaName = "services.rnc"; + static final String hostsXmlSchemaName = "hosts.rnc"; + private final CustomErrorHandler errorHandler = new CustomErrorHandler(); + private final ValidationDriver driver; + private DeployLogger deployLogger; + private static final Logger log = Logger.getLogger(SchemaValidator.class.getName()); + + /** + * Initializes the validator by using the given file as schema file + * @param schema a schema file in RNC format + * @param logger a logger + */ + public SchemaValidator(String schema, DeployLogger logger, Optional<Version> vespaVersion) { + this.deployLogger = logger; + driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); + File schemaDir = new File(schemaDirBase); + try { + schemaDir = saveSchemasFromJar(new File(SchemaValidator.schemaDirBase), vespaVersion); + } catch (IOException e) { + throw new RuntimeException(e); + } + loadSchema(new File(schemaDir + File.separator + "schema" + File.separator + schema)); + IOUtils.recursiveDeleteDir(schemaDir); + } + + /** + * Initializes the validator by using the given file as schema file + * @param schema a schema file in RNC format + * @throws IOException if it is not possible to read schema files + */ + public SchemaValidator(String schema) throws IOException { + this(schema, new BaseDeployLogger(), Optional.empty()); + } + + /** + * Create a validator for services.xml for tests + * @throws IOException if it is not possible to read schema files + */ + public static SchemaValidator createTestValidatorServices() throws IOException { + return new SchemaValidator(servicesXmlSchemaName); + } + + /** + * Create a validator for hosts.xml for tests + * @throws IOException if it is not possible to read schema files + */ + public static SchemaValidator createTestValidatorHosts() throws IOException { + return new SchemaValidator(hostsXmlSchemaName); + } + + private class CustomErrorHandler implements ErrorHandler { + volatile String fileName; + + public void warning(SAXParseException e) throws SAXException { + deployLogger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + private String message(SAXParseException e) { + return "XML error in " + fileName + ": " + + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + } + + /** + * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. + * + * @return the directory the schema files are stored in + * @throws IOException if it is not possible to read schema files + */ + public File saveSchemasFromJar(File tmpBase, Optional<Version> vespaVersion) throws IOException { + final Class<? extends SchemaValidator> schemaValidatorClass = this.getClass(); + final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); + Enumeration<URL> uris = classLoader.getResources("schema"); + if (uris==null) return null; + File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); + log.log(LogLevel.DEBUG, "Saving schemas to " + tmpDir); + while(uris.hasMoreElements()) { + URL u = uris.nextElement(); + log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + if ("jar".equals(u.getProtocol())) { + JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Enumeration<JarEntry> entries = jarFile.entries(); + entries.hasMoreElements();) { + + JarEntry je=entries.nextElement(); + if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { + writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); + } + } + jarFile.close(); + } else if ("bundle".equals(u.getProtocol())) { + Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); + log.log(LogLevel.DEBUG, classLoader.toString()); + log.log(LogLevel.DEBUG, "bundle=" + bundle); + // TODO: Hack to handle cases where bundle=null + if (bundle == null) { + File schemaPath; + if (vespaVersion.isPresent() && vespaVersion.get().getMajor() == 5) { + schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/version/5.x/schema/"); + } else { + schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/"); + } + log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + copySchemas(schemaPath, tmpDir); + } else { + log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + .getVersion())); + for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); + entries.hasMoreElements(); ) { + + URL url = entries.nextElement(); + writeContentsToFile(tmpDir, url.getFile(), url.openStream()); + } + } + } else if ("file".equals(u.getProtocol())) { + File schemaPath = new File(u.getPath()); + copySchemas(schemaPath, tmpDir); + } + } + return tmpDir; + } + + private static void copySchemas(File from, File to) throws IOException { + // TODO: only copy .rnc files. + if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); + if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); + File sourceFile = new File(from, servicesXmlSchemaName); + if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); + IOUtils.copyDirectoryInto(from, to); + } + + private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { + String contents = IOUtils.readAll(new InputStreamReader(inputStream)); + File out = new File(outDir, outFile); + IOUtils.writeFile(out, contents, false); + } + + private void loadSchema(File schemaFile) { + try { + driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); + } catch (SAXException e) { + throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); + } catch (IOException e) { + throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); + } + } + + private PropertyMap instanceProperties() { + PropertyMapBuilder builder = new PropertyMapBuilder(); + builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); + return builder.toPropertyMap(); + } + + public void validate(File file) throws IOException { + validate(file, file.getName()); + } + + public void validate(File file, String fileName) throws IOException { + validate(ValidationDriver.fileInputSource(file), fileName); + } + + public void validate(Reader reader) throws IOException { + validate(new InputSource(reader), null); + } + + public void validate(NamedReader reader) throws IOException { + validate(new InputSource(reader), reader.getName()); + } + + public void validate(InputSource inputSource, String fileName) throws IOException { + errorHandler.fileName = (fileName == null ? " input" : fileName); + try { + if ( ! driver.validate(inputSource)) { + //Shouldn't happen, error handler should have thrown + throw new RuntimeException("Aborting due to earlier XML errors."); + } + } catch (SAXException e) { + //This should never happen, as it is handled by the ErrorHandler + //installed for the driver. + throw new IllegalArgumentException( + "XML error in " + (fileName == null ? " input" : fileName) + ": " + Exceptions.toMessageString(e)); + } + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java new file mode 100644 index 00000000000..80dc01be0b0 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import java.io.IOException; +import java.io.Reader; + +/** + * Simple Validation of services.xml for unit tests against RELAX NG schemas. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class SimpleApplicationValidator { + + public static void checkServices(Reader reader) throws IOException { + SchemaValidator.createTestValidatorServices().validate(reader); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java new file mode 100644 index 00000000000..85ed84c6468 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.defaults.Defaults; +import org.apache.commons.lang.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A global pool of all config definitions that this server knows about. These objects can be shared + * by all tenants, as they are not modified. + * + * @author lulf + * @since 5.10 + */ +public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(StaticConfigDefinitionRepo.class.getName()); + private final Map<ConfigDefinitionKey, ConfigDefinition> configDefinitions = new LinkedHashMap<>(); + private static final String DEFAULT_SERVER_DEF_DIR = Defaults.getDefaults().vespaHome() + "var/db/vespa/config_server/serverdb/classes"; + + public StaticConfigDefinitionRepo() { + this(new File(DEFAULT_SERVER_DEF_DIR)); + } + + public StaticConfigDefinitionRepo(File definitionDir) { + initialize(definitionDir); + } + + private void initialize(File definitionDir) { + if ( ! definitionDir.exists()) return; + + for (File def : definitionDir.listFiles((dir, name) -> name.matches(".*\\.def"))) + addConfigDefinition(def); + } + + private void addConfigDefinition(File def) { + try { + ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); + if (key.getNamespace().isEmpty()) + key = new ConfigDefinitionKey(key.getName(), CNode.DEFAULT_NAMESPACE); + addConfigDefinition(key, def); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Exception adding config definition " + def, e); + } + } + + private void addConfigDefinition(ConfigDefinitionKey key, File defFile) throws IOException { + String payload = IOUtils.readFile(defFile); + configDefinitions.put(key, new ConfigDefinition(key.getName(), StringUtils.split(payload, "\n"))); + } + + @Override + public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { + return Collections.unmodifiableMap(configDefinitions); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java new file mode 100644 index 00000000000..48e80d70312 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.application.provider; + +import com.yahoo.osgi.annotation.ExportPackage; |