diff options
Diffstat (limited to 'config-application-package/src/main/java/com/yahoo/config/model/application')
16 files changed, 2026 insertions, 0 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java b/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java new file mode 100644 index 00000000000..65b3b904c2b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.application; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java new file mode 100644 index 00000000000..6e730ba8410 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.io.File; + +/** + * Definitions of sub-directories of an application package. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class AppSubDirs { + + final Tuple2<File, String> root; + final Tuple2<File, String> templates; + public final Tuple2<File, String> rules; + final Tuple2<File, String> searchchains; + final Tuple2<File, String> docprocchains; + final Tuple2<File, String> routingtables; + final Tuple2<File, String> configDefs; + final Tuple2<File, String> searchdefinitions; + + public AppSubDirs(File root) { + this.root = new Tuple2<>(root, root.getName()); + templates = createTuple(ApplicationPackage.TEMPLATES_DIR); + rules = createTuple(ApplicationPackage.RULES_DIR.getRelative()); + searchchains = createTuple(ApplicationPackage.SEARCHCHAINS_DIR); + docprocchains = createTuple(ApplicationPackage.DOCPROCCHAINS_DIR); + routingtables = createTuple(ApplicationPackage.ROUTINGTABLES_DIR); + configDefs = createTuple(ApplicationPackage.CONFIG_DEFINITIONS_DIR); + searchdefinitions = createTuple(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + } + + private Tuple2<File, String> createTuple(String name) { + return new Tuple2<>(file(name), name); + } + + public File file(String subPath) { + return new File(root.first, subPath); + } + + public File root() { + return root.first; + } + + public File templates() { + return templates.first; + } + + public File rules() { + return rules.first; + } + + public File searchchains() { + return searchchains.first; + } + + public File docprocchains() { + return docprocchains.first; + } + + public File configDefs() { + return configDefs.first; + } + + public File searchdefinitions() { + return searchdefinitions.first; + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java new file mode 100644 index 00000000000..16c3ef3e029 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.Version; +import com.yahoo.path.Path; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +/** + * Validation of xml files in application package against RELAX NG schemas. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class ApplicationPackageXmlFilesValidator { + + private final AppSubDirs appDirs; + private final DeployLogger logger; + private final Optional<Version> vespaVersion; + + private static final FilenameFilter xmlFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".xml"); + } + }; + + public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, DeployLogger logger, Optional<Version> vespaVersion) { + this.appDirs = appDirs; + this.logger = logger; + this.vespaVersion = vespaVersion; + } + + public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, DeployLogger logger, Optional<Version> vespaVersion) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), logger, vespaVersion); + } + + public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), new BaseDeployLogger(), Optional.<Version>empty()); + } + + // Verify that files a and b does not coexist. + private void checkConflicts(String a, String b) throws IllegalArgumentException { + if (appDirs.file(a).exists() && appDirs.file(b).exists()) + throw new IllegalArgumentException("Application package in " + appDirs.root() + " contains both " + a + " and " + b + + ", please use just one of them"); + } + + @SuppressWarnings("deprecation") + public void checkApplication() throws IOException { + validateHostsFile(SchemaValidator.hostsXmlSchemaName); + validateServicesFile(SchemaValidator.servicesXmlSchemaName); + + if (appDirs.searchdefinitions().exists()) { + if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { + throw new IllegalArgumentException("Application package in " + appDirs.root() + + " must contain at least one search definition (.sd) file when directory searchdefinitions/ exists."); + } + } + + validate(appDirs.routingtables, "routing-standalone.rnc"); + } + + // For testing + public static void checkIncludedDirs(ApplicationPackage app) throws IOException { + for (String includedDir : app.getUserIncludeDirs()) { + List<NamedReader> includedFiles = app.getFiles(Path.fromString(includedDir), ".xml", true); + for (NamedReader file : includedFiles) { + createSchemaValidator("container-include.rnc", Optional.empty()).validate(file); + } + } + } + + @SuppressWarnings("deprecation") + private void validateHostsFile(String hostsXmlSchemaName) throws IOException { + if (appDirs.file(FilesApplicationPackage.HOSTS).exists()) { + validate(hostsXmlSchemaName, FilesApplicationPackage.HOSTS); + } + + } + + private void validateServicesFile(String servicesXmlSchemaName) throws IOException { + // vespa-services.xml or services.xml. Fallback to vespa-services.xml + validate(servicesXmlSchemaName, servicesFileName()); + } + + private void validate(String schemaName, String xmlFileName) throws IOException { + createSchemaValidator(schemaName, vespaVersion).validate(appDirs.file(xmlFileName)); + } + + @SuppressWarnings("deprecation") + private String servicesFileName() { + String servicesFile = FilesApplicationPackage.SERVICES; + if (!appDirs.file(servicesFile).exists()) { + throw new IllegalArgumentException("Application package in " + appDirs.root() + + " must contain " + FilesApplicationPackage.SERVICES); + } + return servicesFile; + } + + private void validate(Tuple2<File, String> directory, String schemaFile) throws IOException { + if ( ! directory.first.isDirectory()) return; + validate(directory, createSchemaValidator(schemaFile, vespaVersion)); + } + + private void validate(Tuple2<File, String> directory, SchemaValidator validator) throws IOException { + File dir = directory.first; + if ( ! dir.isDirectory()) return; + + String directoryName = directory.second; + for (File f : dir.listFiles(xmlFilter)) { + if (f.isDirectory()) + validate(new Tuple2<>(f, directoryName + File.separator + f.getName()),validator); + else + validator.validate(f, directoryName + File.separator + f.getName()); + } + } + + private static SchemaValidator createSchemaValidator(String schemaFile, Optional<Version> vespaVersion) { + return new SchemaValidator(schemaFile, new BaseDeployLogger(), vespaVersion); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java new file mode 100644 index 00000000000..50412348893 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/BaseDeployLogger.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.api.DeployLogger; + +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * Only logs to a normal {@link Logger} + * @author vegardh + * + */ +public final class BaseDeployLogger implements DeployLogger { + + private static final Logger log = Logger.getLogger("DeployLogger"); + + @Override + public final void log(Level level, String message) { + log.log(level, message); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java new file mode 100644 index 00000000000..8f2026afc66 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.google.common.base.Charsets; +import com.yahoo.collections.Tuple2; +import com.yahoo.config.codegen.CNode; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.*; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; + +/** + * A Bundle represents an OSGi bundle inside the model, and provides utilities + * for accessing resources within that bundle. + * + * @author tonytv, lulf + * @since 5.1 + */ +public class Bundle { + private static final Logger log = Logger.getLogger(Bundle.class.getName()); + private static final String DEFPATH = "configdefinitions/"; // path inside jar file + private final File bundleFile; + private final JarFile jarFile; + private final List<DefEntry> defEntries; + + public Bundle(JarFile jarFile, File bundleFile) { + this.jarFile = jarFile; + this.bundleFile = bundleFile; + defEntries = findDefEntries(); + } + + public static List<Bundle> getBundles(File bundleDir) { + try { + List<Bundle> bundles = new ArrayList<>(); + for (File bundleFile : getBundleFiles(bundleDir)) { + JarFile jarFile; + try { + jarFile = new JarFile(bundleFile); + } catch (ZipException e) { + throw new IllegalArgumentException("Error opening jar file '" + bundleFile.getName() + + "'. Please check that this is a valid jar file"); + } + bundles.add(new Bundle(jarFile, bundleFile)); + } + return bundles; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static List<File> getBundleFiles(File bundleDir) { + if (!bundleDir.isDirectory()) { + return new ArrayList<>(); + } + return Arrays.asList(bundleDir.listFiles( + new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + } + })); + } + + public List<DefEntry> getDefEntries() { + return Collections.unmodifiableList(defEntries); + } + + /** + * Returns a list of all .def-file entries in this Component. + * @return A list of .def-file entries. + */ + private List<DefEntry> findDefEntries() { + List<DefEntry> defEntries = new ArrayList<>(); + + ZipEntry defDir = jarFile.getEntry(DEFPATH); + + if ((defDir == null) || !defDir.isDirectory()) + return defEntries; + + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + + if (name.endsWith(".def")) { + if (name.matches("^" + DEFPATH + ".*\\.def$")) { + defEntries.add(new DefEntry(this, entry)); + } else { + log.info("Config definition file '" + name + "' in component '" + jarFile.getName() + + "' will not be used. Files must reside in the '" + DEFPATH + + "' directory in the .jar file"); + + } + } + } + return defEntries; + } + + public JarFile getJarFile() { + return jarFile; + } + + public File getFile() { + return bundleFile; + } + + /** + * Represents a def-file inside a Component. Immutable. + */ + public static class DefEntry { + + private final Bundle bundle; + private final ZipEntry zipEntry; + public final String defName; // Without version number and suffix. + public final String defNamespace; + public final String contents; + + /** + * @param bundle The bundle this def entry belongs to. + * @param zipEntry The ZipEntry representing the def-file. + */ + public DefEntry(Bundle bundle, ZipEntry zipEntry) { + this.bundle = bundle; + this.zipEntry = zipEntry; + + String entryName = zipEntry.getName(); + Tuple2<String, String> nameAndNamespace = ConfigUtils.getNameAndNamespaceFromString(entryName.substring(DEFPATH.length(), entryName.indexOf(".def"))); + + defName = nameAndNamespace.first; + defNamespace = getNamespace(); + if (defNamespace.isEmpty()) + throw new IllegalArgumentException("Config definition '" + defName + "' is missing a namespace"); + contents = getContents(); + } + + /** + * Returns the namespace of the .def-file, as given by the "namespace=" statement inside the given entry. + * @return The namespace string, or "" (empty string) if no namespace exists + */ + private String getNamespace() { + return ConfigUtils.getDefNamespace(getReader()); + } + + private String getContents() { + StringBuilder ret = new StringBuilder(""); + BufferedReader reader = new BufferedReader(getReader()); + try { + String str = reader.readLine(); + while (str != null){ + ret.append(str); + str = reader.readLine(); + if (str != null) { + ret.append("\n"); + } + } + reader.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Failed reading contents of def-file '" + defName + + ".def in component " + bundle.jarFile.getName(),e); + } + return ret.toString(); + } + + public Reader getReader() { + if (zipEntry == null) { + return new StringReader(""); + } + try { + return new InputStreamReader(bundle.jarFile.getInputStream(zipEntry), Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException("IOException", e); + } + } + + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java new file mode 100644 index 00000000000..25aacdfd74b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +/** + * A class for holding values generated or computed during deployment + * + * @author musum + * @since 5.1.11 + */ +public class DeployData { + /* Which user deployed */ + private final String deployedByUser; + + /* Name of application given by user */ + private final String applicationName; + + /* The absolute path to the directory holding the application */ + private final String deployedFromDir; + + /* Timestamp when a deployment was made */ + private final long deployTimestamp; + + /* Application generation. Incremented by one each time an application is deployed. */ + private final long generation; + private final long currentlyActiveGeneration; + + public DeployData(String deployedByUser, String deployedFromDir, String applicationName, Long deployTimestamp, Long generation, long currentlyActiveGeneration) { + this.deployedByUser = deployedByUser; + this.deployedFromDir = deployedFromDir; + this.applicationName = applicationName; + this.deployTimestamp = deployTimestamp; + this.generation = generation; + this.currentlyActiveGeneration = currentlyActiveGeneration; + } + + public String getDeployedByUser() { + return deployedByUser; + } + + public String getDeployedFromDir() { + return deployedFromDir; + } + + public long getDeployTimestamp() { + return deployTimestamp; + } + + public long getGeneration() { + return generation; + } + + public long getCurrentlyActiveGeneration() { + return currentlyActiveGeneration; + } + + public String getApplicationName() { + return applicationName; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java new file mode 100644 index 00000000000..735b8111f02 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; + +import java.lang.reflect.Constructor; + +/** + * Convenience for creating a {@link com.yahoo.config.FileReference}. + * + * @author gjoranv + */ +public class FileReferenceCreator { + + public static FileReference create(String stringVal) { + try { + Constructor<FileReference> ctor = FileReference.class.getDeclaredConstructor(String.class); + ctor.setAccessible(true); + return ctor.newInstance(stringVal); + } catch (Exception e) { + throw new RuntimeException("Could not create a new " + FileReference.class.getName() + + ". This should never happen!", e); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java new file mode 100644 index 00000000000..67a180d334d --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author lulf + * @author vegardh + * @since 5.1 + */ +public class FilesApplicationFile extends ApplicationFile { + private static final Logger log = Logger.getLogger("FilesApplicationFile"); + private final File file; + private final ObjectMapper mapper = new ObjectMapper(); + public FilesApplicationFile(Path path, File file) { + super(path); + this.file = file; + } + + @Override + public boolean isDirectory() { + return file.isDirectory(); + } + + @Override + public boolean exists() { + return file.exists(); + } + + @Override + public ApplicationFile delete() { + log.log(LogLevel.DEBUG, "Delete " + file); + if (file.isDirectory() && !listFiles().isEmpty()) { + throw new RuntimeException("files. Can't delete, directory not empty: " + this + "(" + listFiles() + ")." + listFiles().size()); + } + if (file.isDirectory() && file.listFiles() != null && file.listFiles().length > 0) { + for (File f : file.listFiles()) { + deleteFile(f); + } + } + if (!file.delete()) { + throw new IllegalStateException("Unable to delete: "+this); + } + try { + writeMetaFile("", ApplicationFile.ContentStatusDeleted); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + public static boolean deleteFile(File path) { + if( path.exists() ) { + if (path.isDirectory()) { + File[] files = path.listFiles(); + for(int i=0; i<files.length; i++) { + if(files[i].isDirectory()) { + deleteFile(files[i]); + } else { + files[i].delete(); + } + } + } + } + return(path.delete()); + } + + @Override + public Reader createReader() throws FileNotFoundException { + return new FileReader(file); + } + + @Override + public InputStream createInputStream() throws FileNotFoundException { + return new FileInputStream(file); + } + + @Override + public ApplicationFile createDirectory() { + if (file.isDirectory()) return this; + if (file.exists()) { + throw new IllegalArgumentException("Unable to create directory, file exists: "+file); + } + if (!file.mkdirs()) { + throw new IllegalArgumentException("Unable to create directory: "+file); + } + try { + writeMetaFile("", ApplicationFile.ContentStatusNew); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public ApplicationFile writeFile(Reader input) { + if (file.getParentFile() != null) { + file.getParentFile().mkdirs(); + } + try { + String data = com.yahoo.io.IOUtils.readAll(input); + String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew; + IOUtils.writeFile(file, data, false); + writeMetaFile(data, status); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public List<ApplicationFile> listFiles(final PathFilter filter) { + List<ApplicationFile> files = new ArrayList<>(); + if (!file.isDirectory()) { + return files; + } + FileFilter fileFilter = pathname -> filter.accept(path.append(pathname.getName())); + for (File child : file.listFiles(fileFilter)) { + // Ignore dot-files. + if (!child.getName().startsWith(".")) { + files.add(new FilesApplicationFile(path.append(child.getName()), child)); + } + } + return files; + } + + private void writeMetaFile(String data, String status) throws IOException { + File metaDir = createMetaDir(); + log.log(LogLevel.DEBUG, "meta dir=" + metaDir); + File metaFile = new File(metaDir + "/" + getPath().getName()); + if (status == null) { + status = ApplicationFile.ContentStatusNew; + if (metaFile.exists()) { + status = ApplicationFile.ContentStatusChanged; + } + } + String hash; + if (file.isDirectory() || status.equals(ApplicationFile.ContentStatusDeleted)) { + hash = ""; + } else { + hash = ConfigUtils.getMd5(data); + } + mapper.writeValue(metaFile, new MetaData(status, hash)); + } + + private File createMetaDir() { + File metaDir = getMetaDir(); + if (!metaDir.exists()) { + log.log(LogLevel.DEBUG, "Creating meta dir " + metaDir); + metaDir.mkdirs(); + } + return metaDir; + } + + private File getMetaDir() { + String substring = file.getAbsolutePath().substring(0, file.getAbsolutePath().lastIndexOf("/") + 1); + return new File(substring + Path.fromString(".meta/")); + } + + public MetaData getMetaData() { + File metaDir = getMetaDir(); + File metaFile = new File(metaDir + "/" + getPath().getName()); + log.log(LogLevel.DEBUG, "Getting metadata for " + metaFile); + if (metaFile.exists()) { + try { + return mapper.readValue(metaFile, MetaData.class); + } catch (IOException e) { + System.out.println("whot:" + Exceptions.toMessageString(e)); + // return below + } + } + try { + if (file.isDirectory()) { + return new MetaData(ApplicationFile.ContentStatusNew, ""); + } else { + return new MetaData(ApplicationFile.ContentStatusNew, ConfigUtils.getMd5(IOUtils.readAll(createReader()))); + } + } catch (IOException | IllegalArgumentException e) { + return null; + } + } + + @Override + public int compareTo(ApplicationFile other) { + if (other == this) return 0; + return this.getPath().getName().compareTo((other).getPath().getName()); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java new file mode 100644 index 00000000000..21e7de8fa7b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -0,0 +1,772 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.ConfigDefinitionDir; +import com.yahoo.config.application.Xml; +import com.yahoo.config.application.XmlPreProcessor; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.RuleConfigDeriver; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.path.Path; +import com.yahoo.io.HexDump; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.util.ConfigUtils; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.*; +import java.net.JarURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.security.MessageDigest; +import java.util.*; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.io.IOUtils.readAll; +import static com.yahoo.text.Lowercase.toLowerCase; + + +/** + * Application package derived from local files, ie. during deploy. + * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or + * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}. + * + * @author vegardh + */ +public class FilesApplicationPackage implements ApplicationPackage { + + private static final Logger log = Logger.getLogger(FilesApplicationPackage.class.getName()); + private static final String META_FILE_NAME = ".applicationMetaData"; + + private final File appDir; + private final File preprocessedDir; + private final File configDefsDir; + private final AppSubDirs appSubDirs; + // NOTE: these directories exist in the original user app, but their locations are given in 'services.xml' + private final List<String> userIncludeDirs = new ArrayList<>(); + private final ApplicationMetaData metaData; + + /** + * Returns an application package object based on the given application dir + * + * @param appDir application package directory + * @return an Application package instance + */ + public static FilesApplicationPackage fromFile(File appDir) { + return new Builder(appDir).preprocessedDir(new File(appDir, ".preprocessed")).build(); + } + + /** Creates package from a local directory, typically deploy app */ + public static FilesApplicationPackage fromFileWithDeployData(File appDir, DeployData deployData) { + return new Builder(appDir).deployData(deployData).build(); + } + + /** + * Builder for {@link com.yahoo.config.model.application.provider.FilesApplicationPackage}. Use + * this to create instances in a flexible manner. + */ + public static class Builder { + private final File appDir; + private Optional<File> preprocessedDir = Optional.empty(); + private Optional<ApplicationMetaData> metaData = Optional.empty(); + + public Builder(File appDir) { + this.appDir = appDir; + } + + public Builder preprocessedDir(File preprocessedDir) { + this.preprocessedDir = Optional.ofNullable(preprocessedDir); + return this; + } + + public Builder deployData(DeployData deployData) { + this.metaData = Optional.of(metaDataFromDeployData(appDir, deployData)); + return this; + } + + public FilesApplicationPackage build() { + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, ".preprocessed")), metaData.orElse(readMetaData(appDir))); + } + } + + private static ApplicationMetaData metaDataFromDeployData(File appDir, DeployData deployData) { + return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), deployData.getDeployTimestamp(), + deployData.getApplicationName(), computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); + } + + /** + * New package from given path on local file system. Retrieves config definition files from + * the default location 'serverdb/classes'. + * + * @param appDir application package directory + * @param preprocessedDir preprocessed application package output directory + * @param metaData metadata for this application package + */ + @SuppressWarnings("deprecation") + private FilesApplicationPackage(File appDir, File preprocessedDir, ApplicationMetaData metaData) { + verifyAppDir(appDir); + this.appDir = appDir; + this.preprocessedDir = preprocessedDir; + appSubDirs = new AppSubDirs(appDir); + configDefsDir = new File(appDir, ApplicationPackage.CONFIG_DEFINITIONS_DIR); + addUserIncludeDirs(); + this.metaData = metaData; + } + + public String getApplicationName() { + return metaData.getApplicationName(); + } + + @Override + public List<NamedReader> getFiles(Path relativePath, String suffix, boolean recurse) { + return getFiles(relativePath, "", suffix, recurse); + } + + @Override + public ApplicationFile getFile(Path path) { + File file = (path.isRoot() ? appDir : new File(appDir, path.getRelative())); + return new FilesApplicationFile(path, file); + } + + @Override + public ApplicationMetaData getMetaData() { + return metaData; + } + + private List<NamedReader> getFiles(Path relativePath,String namePrefix,String suffix,boolean recurse) { + try { + List<NamedReader> readers=new ArrayList<>(); + File dir = new File(appDir, relativePath.getRelative()); + if ( ! dir.isDirectory()) return readers; + + final File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + if (recurse) + readers.addAll(getFiles(relativePath.append(file.getName()), namePrefix + "/" + file.getName(), suffix, recurse)); + } else { + if (suffix == null || file.getName().endsWith(suffix)) + readers.add(new NamedReader(file.getName(), new FileReader(file))); + } + } + } + return readers; + } + catch (IOException e) { + throw new RuntimeException("Could not open (all) files in '" + relativePath + "'",e); + } + } + + private void verifyAppDir(File appDir) { + if (appDir==null || !appDir.isDirectory()) { + throw new IllegalArgumentException("Path '" + appDir + "' is not a directory."); + } + if (! appDir.canRead()){ + throw new IllegalArgumentException("Cannot read from application directory '" + appDir + "'"); + } + } + + @Override + public Reader getHosts() { + try { + File hostsFile = getHostsFile(); + if (!hostsFile.exists()) return null; + return new FileReader(hostsFile); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public String getHostSource() { + return getHostsFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getHostsFile() { + return new File(appDir, ApplicationPackage.HOSTS); + } + + private File getFileWithFallback(String first, String second) { + File firstFile = new File(appDir, first); + File secondFile = new File(appDir, second); + if (firstFile.exists()) { + return firstFile; + } else if (secondFile.exists()) { + return secondFile; + } else { + return firstFile; + } + } + + @Override + public String getServicesSource() { + return getServicesFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getServicesFile() { + return new File(appDir, ApplicationPackage.SERVICES); + } + + @Override + public Optional<Reader> getDeployment() { return optionalFile(DEPLOYMENT_FILE); } + + @Override + public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES); } + + private Optional<Reader> optionalFile(Path filePath) { + try { + return Optional.of(getFile(filePath).createReader()); + } catch (FileNotFoundException e) { + return Optional.empty(); + } + } + + + @Override + public List<String> getUserIncludeDirs() { + return Collections.unmodifiableList(userIncludeDirs); + } + + public void addUserIncludeDirs() { + Document services; + try { + services = Xml.getDocument(getServices()); + } catch (Exception e) { + return; // This method does not validate that services.xml exists, or that it is valid xml. + } + NodeList includeNodes = services.getElementsByTagName(IncludeDirs.INCLUDE); + + for (int i=0; i < includeNodes.getLength(); i++) { + Node includeNode = includeNodes.item(i); + addIncludeDir(includeNode); + } + } + + private void addIncludeDir(Node includeNode) { + if (! (includeNode instanceof Element)) + return; + Element include = (Element) includeNode; + if (! include.hasAttribute(IncludeDirs.DIR)) + return; + String dir = include.getAttribute(IncludeDirs.DIR); + validateIncludeDir(dir); + IncludeDirs.validateFilesInIncludedDir(dir, include.getParentNode(), this); + log.log(LogLevel.INFO, "Adding user include dir '" + dir + "'"); + userIncludeDirs.add(dir); + } + + @Override + public void validateIncludeDir(String dirName) { + IncludeDirs.validateIncludeDir(dirName, this); + } + + @Override + public Collection<NamedReader> searchDefinitionContents() { + Map<String, NamedReader> ret = new LinkedHashMap<>(); + Set<String> fileSds = new LinkedHashSet<>(); + Set<String> bundleSds = new LinkedHashSet<>(); + try { + for (File f : getSearchDefinitionFiles()) { + fileSds.add(f.getName()); + ret.put(f.getName(), new NamedReader(f.getName(), new FileReader(f))); + } + for (Map.Entry<String, String> e : allSdsFromDocprocBundlesAndClasspath(appDir).entrySet()) { + bundleSds.add(e.getKey()); + ret.put(e.getKey(), new NamedReader(e.getKey(), new StringReader(e.getValue()))); + } + } catch (Exception e) { + throw new IllegalArgumentException("Couldn't get search definition contents.", e); + } + verifySdsDisjoint(fileSds, bundleSds); + return ret.values(); + } + + /** + * Verify that two sets of search definitions are disjoint (TODO: everything except error message is very generic). + * @param fileSds Set of search definitions from file + * @param bundleSds Set of search definitions from bundles + */ + private void verifySdsDisjoint(Set<String> fileSds, Set<String> bundleSds) { + if (!Collections.disjoint(fileSds, bundleSds)) { + Collection<String> disjoint = new ArrayList<>(fileSds); + disjoint.retainAll(bundleSds); + throw new IllegalArgumentException("For the following search definitions names there are collisions between those specified inside " + + "docproc bundles and those in searchdefinitions/ in application package: "+disjoint); + } + } + + /** + * Returns sdName→payload for all SDs in all docproc bundles and on local classpath. + * Throws {@link IllegalArgumentException} if there are multiple sd files of same name. + * @param appDir application package directory + * @return a map from search definition name to search definition content + * @throws IOException if reading a search definition fails + */ + public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { + File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); + if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); + List<String> usedNames = new ArrayList<>(); + Map<String, String> ret = new LinkedHashMap<>(); + + // try classpath first + allSdsOnClassPath(usedNames, ret); + + for (File bundle : dpChains.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + }})) { + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+ + bundle.getName()+"' is already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + return ret; + } + + private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException { + Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + + while(resources.hasMoreElements()) { + URL resource = resources.nextElement(); + + String protocol = resource.getProtocol(); + + if ("file".equals(protocol)) { + File file; + try { + file = new File(resource.toURI()); + } catch (URISyntaxException e) { + continue; + } + // only interested in directories + if (file.isDirectory()) { + List<File> sdFiles = getSearchDefinitionFiles(file); + for (File sdFile : sdFiles) { + String sdName = sdFile.getName(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+ + "' found in classpath already used earlier in classpath."); + } + usedNames.add(sdName); + String contents = IOUtils.readAll(new FileReader(sdFile)); + ret.put(sdFile.getName(), contents); + } + } + } + else if ("jar".equals(protocol)) { + JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definitions name '"+sdName+ + "' used in bundle '"+jarFile.getName()+"' already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + } + } + + private Reader retrieveConfigDefReader(String defName) { + File def = new File(configDefsDir + File.separator + defName); + try { + return new NamedReader(def.getAbsolutePath(), new FileReader(def)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read config definition file '" + + def.getAbsolutePath() + "'", e); + } + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs = new LinkedHashMap<>(); + + if (configDefsDir.isDirectory()) { + addAllDefsFromConfigDir(defs, configDefsDir); + } + addAllDefsFromBundles(defs, FilesApplicationPackage.getComponents(appDir)); + return defs; + } + + private void addAllDefsFromBundles(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, List<Component> components) { + for (Component component : components) { + Bundle bundle = component.getBundle(); + for (final Bundle.DefEntry def : bundle.getDefEntries()) { + final ConfigDefinitionKey defKey = new ConfigDefinitionKey(def.defName, def.defNamespace); + if (!defs.containsKey(defKey)) { + defs.put(defKey, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(defKey.getName(), new StringReader(def.contents)); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return def.contents; + } + }); + } + } + } + } + + private void addAllDefsFromConfigDir(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, File configDefsDir) { + log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); + for (final File def : configDefsDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\.def");}})) { + + log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'"); + String[] nv = def.getName().split("\\.def"); + if (nv == null) { + log.log(LogLevel.WARNING, "Skipping '" + def + "', cannot determine name"); + } else { + ConfigDefinitionKey key; + try { + key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); + } catch (IOException e) { + e.printStackTrace(); + break; + } + if (key.getNamespace().isEmpty()) { + throw new IllegalArgumentException("Config definition '" + nv + "' has no namespace"); + } + boolean addFile = false; + if (defs.containsKey(key)) { + if (nv[0].contains(".")) { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". The file '" + def + "' will take precedence"); + addFile = true; + } else { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". Skipping '" + def + "', as it does not contain namespace in filename"); + } + } else { + addFile = true; + } + if (addFile) { + log.log(LogLevel.DEBUG, "Adding " + key + " to archive of all existing config defs"); + final ConfigDefinitionKey finalKey = key; + defs.put(key, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(finalKey.getName(), retrieveConfigDefReader(def.getName())); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return readConfigDefinition(def.getName()); + } + }); + } + } + } + } + + private String readConfigDefinition(String name) { + try (Reader reader = retrieveConfigDefReader(name)) { + return IOUtils.readAll(reader); + } catch (IOException e) { + throw new RuntimeException("Error reading config definition " + name, e); + } + } + + @Override + public Reader getServices() { + try { + return new FileReader(getServicesSource()); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + //Only intended for DeployProcessor, others should use the member version + static List<File> getSearchDefinitionFiles(File appDir) { + //The dot is escaped later in this method: + assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.'); + + List<File> ret = new ArrayList<>(); + File sdDir; + + sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + if (!sdDir.isDirectory()) { + return ret; + } + ret.addAll(Arrays.asList( + sdDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}}))); + return ret; + } + + public List<File> getSearchDefinitionFiles() { + return getSearchDefinitionFiles(appDir); + } + + //Only for use by deploy processor + public static List<Component> getComponents(File appDir) { + List<Component> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new Component(bundle, new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath()))); + } + return components; + } + + private static List<ComponentInfo> getComponentsInfo(File appDir) { + List<ComponentInfo> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath())); + } + return components; + } + + @Override + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + return getComponentsInfo(appDir); + } + + /** + * Returns a list of all components in this package. + * + * @return A list of components. + */ + public List<Component> getComponents() { + return getComponents(appDir); + } + + public File getAppDir() throws IOException { + return appDir.getCanonicalFile(); + } + + public static ApplicationMetaData readMetaData(File appDir) { + ApplicationMetaData defaultMetaData = new ApplicationMetaData(appDir, "n/a", "n/a", 0l, "", 0l, 0l); + File metaFile = new File(appDir, META_FILE_NAME); + if (!metaFile.exists()) { + return defaultMetaData; + } + try (FileReader reader = new FileReader(metaFile)) { + return ApplicationMetaData.fromJsonString(IOUtils.readAll(reader)); + } catch (Exception e) { + // Not a big deal, return default + return defaultMetaData; + } + } + + /** + * Represents a component in the application package. Immutable. + */ + public static class Component { + + public final ComponentInfo info; + private final Bundle bundle; + + public Component(Bundle bundle, ComponentInfo info) { + this.bundle = bundle; + this.info = info; + } + + public List<Bundle.DefEntry> getDefEntries() { + return bundle.getDefEntries(); + } + + public Bundle getBundle() { + return bundle; + } + } // class Component + + /** + * Reads a ranking expression from file to a string and returns it. + * + * @param name the name of the file to return, either absolute or + * relative to the search definition directory in the application package + * @return the content of a ranking expression file + * @throws IllegalArgumentException if the file was not found or could not be read + */ + // TODO: A note on absolute paths: We don't want to support this and it should be removed on 6.0 + // Currently one system test (basicmlr) depends on it. + @Override + public Reader getRankingExpression(String name) { + try { + return IOUtils.createReader(expressionFileNameToFile(name), "utf-8"); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read ranking expression file '" + name + "'", e); + } + } + + private File expressionFileNameToFile(String name) { + File expressionFile = new File(name); + if (expressionFile.isAbsolute()) return expressionFile; + + File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + return new File(sdDir, name); + } + + @Override + public File getFileReference(Path pathRelativeToAppDir) { + return new File(appDir, pathRelativeToAppDir.getRelative()); + } + + @Override + public void validateXML(DeployLogger logger) throws IOException { + validateXML(logger, Optional.empty()); + } + + @Override + public void validateXML(DeployLogger logger, Optional<Version> vespaVersion) throws IOException { + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, logger, vespaVersion); + xmlFilesValidator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); + } + + @Override + public void writeMetaData() throws IOException { + File metaFile = new File(appDir, META_FILE_NAME); + IOUtils.writeFile(metaFile, metaData.asJsonString(), false); + } + + @Override + public Collection<NamedReader> getSearchDefinitions() { + return searchDefinitionContents(); + } + + private void preprocessXML(File destination, File inputXml, Zone zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { + Document document = new XmlPreProcessor(appDir, inputXml, zone.environment(), zone.region()).run(); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + try (FileOutputStream outputStream = new FileOutputStream(destination)) { + transformer.transform(new DOMSource(document), new StreamResult(outputStream)); + } + } + + @Override + public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { + IOUtils.recursiveDeleteDir(preprocessedDir); + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> !name.equals(".preprocessed") && + !name.equals(SERVICES) && + !name.equals(HOSTS) && + !name.equals(CONFIG_DEFINITIONS_DIR)); + preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); + if (getHostsFile().exists()) { + preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); + } + FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir); + preprocessed.copyUserDefsIntoApplication(); + return preprocessed; + } + + private void copyUserDefsIntoApplication() { + File destination = appSubDirs.configDefs(); + destination.mkdir(); + ConfigDefinitionDir defDir = new ConfigDefinitionDir(destination); + // Copy the user's def files from components. + List<Bundle> bundlesAdded = new ArrayList<>(); + for (FilesApplicationPackage.Component component : FilesApplicationPackage.getComponents(appSubDirs.root())) { + Bundle bundle = component.getBundle(); + defDir.addConfigDefinitionsFromBundle(bundle, bundlesAdded); + bundlesAdded.add(bundle); + } + } + + /** + * Computes an md5 hash of the contents of the application package + * + * @return an md5sum of the application package + */ + private static String computeCheckSum(File appDir) { + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + for (File file : appDir.listFiles((dir, name) -> !name.equals(ApplicationPackage.EXT_DIR) && !name.startsWith("."))) { + addPathToDigest(file, "", md5, true, false); + } + return toLowerCase(HexDump.toHexString(md5.digest())); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Adds the given path to the digest, or does nothing if path is neither file nor dir + * @param path path to add to message digest + * @param suffix only files with this suffix are considered + * @param digest the {link @MessageDigest} to add the file paths to + * @param recursive whether to recursively find children in the paths + * @param fullPathNames Whether to include the full paths in checksum or only the names + * @throws java.io.IOException if adding path to digest fails when reading files from path + */ + private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException { + if (!path.exists()) return; + if (fullPathNames) { + digest.update(path.getPath().getBytes(Utf8.getCharset())); + } else { + digest.update(path.getName().getBytes(Utf8.getCharset())); + } + if (path.isFile()) { + FileInputStream is = new FileInputStream(path); + addToDigest(is, digest); + is.close(); + } else if (path.isDirectory()) { + final File[] files = path.listFiles(); + if (files != null) { + for (File elem : files) { + if ((elem.isDirectory() && recursive) || elem.getName().endsWith(suffix)) { + addPathToDigest(elem, suffix, digest, recursive, fullPathNames); + } + } + } + } + } + + private static final int MD5_BUFFER_SIZE = 65536; + private static void addToDigest(InputStream is, MessageDigest digest) throws IOException { + if (is==null) return; + byte[] buffer = new byte[MD5_BUFFER_SIZE]; + int i; + do { + i=is.read(buffer); + if (i > 0) { + digest.update(buffer, 0, i); + } + } while(i!=-1); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java new file mode 100644 index 00000000000..8d0707f9e99 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/IncludeDirs.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.io.File; +import java.util.List; + +/** + * Helper methods for directories included from services.xml in a <include dir=''/> element. + * + * @author gjoranv + * @since 5.1.19 + */ +public class IncludeDirs { + + public static final String INCLUDE = "include"; + public static final String DIR = "dir"; + + private IncludeDirs() { + throw new UnsupportedOperationException(IncludeDirs.class.getName() + " cannot be instantiated!"); + } + + public static void validateIncludeDir(String dirName, FilesApplicationPackage app) { + File file = new File(dirName); + + if (file.isAbsolute()) { + throw new IllegalArgumentException("Cannot include directory '" + dirName + + "', absolute paths are not supported. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + + file = app.getFileReference(Path.fromString(dirName)); + + if (!file.exists()) { + throw new IllegalArgumentException("Cannot include directory '" + dirName + + "', as it does not exist. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + + if (!file.isDirectory()) { + throw new IllegalArgumentException("Cannot include '" + dirName + + "', as it is not a directory. Directory must reside in application package, " + + "and path must be given relative to application package."); + } + } + + + public static void validateFilesInIncludedDir(String dirName, Node parentNode, ApplicationPackage app) { + if (! (parentNode instanceof Element)) { + throw new IllegalStateException("The parent xml node of an include is not an Element: " + parentNode); + } + String parentTagName = ((Element) parentNode).getTagName(); + + List<Element> includedRootElems = Xml.allElemsFromPath(app, dirName); + for (Element includedRootElem : includedRootElems) { + validateIncludedFile(includedRootElem, parentTagName, dirName); + } + } + + /** + * @param includedRootElem The root element of the included file + * @param dirName The name of the included dir + */ + private static void validateIncludedFile(Element includedRootElem, String parentTagName, String dirName) { + if (!parentTagName.equals(includedRootElem.getTagName())) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' does not have <" + parentTagName + "> as root element."); + } + if (includedRootElem.hasAttributes()) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' has attributes set on its root element <" + parentTagName + + ">. These must be set in services.xml instead."); + } + if (XML.getChild(includedRootElem, INCLUDE) != null) { + throw new IllegalArgumentException("File included from '<include dir\"" + dirName + + "\">' has <include> subelement. Recursive inclusion is not supported."); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java new file mode 100644 index 00000000000..334fda6e6eb --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * A file registry for testing, and, it seems, doubling as a null registry in some code paths. + * + * @author tonytv + */ +public class MockFileRegistry implements FileRegistry { + + public FileReference addFile(String relativePath) { + return FileReferenceCreator.create("0123456789abcdef"); + } + + @Override + public String fileSourceHost() { + return "localhost.fortestingpurposesonly"; + } + + public static final Entry entry1 = new Entry("component/path1", FileReferenceCreator.create("1234")); + public static final Entry entry2 = new Entry("component/path2", FileReferenceCreator.create("56789")); + + public List<Entry> export() { + List<Entry> result = new ArrayList<>(); + result.add(entry1); + result.add(entry2); + return result; + } + + @Override + public Set<String> allRelativePaths() { + return Collections.emptySet(); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java new file mode 100644 index 00000000000..67a24e0159b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.*; +import java.util.regex.Pattern; + +/** + * Registry of files added earlier (i.e. during deployment) + * + * @author tonytv + */ +public class PreGeneratedFileRegistry implements FileRegistry { + + private final String fileSourceHost; + private final Map<String, String> path2Hash = new LinkedHashMap<>(); + + private static String entryDelimiter = "\t"; + private static Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); + + private PreGeneratedFileRegistry(Reader readerArg) { + BufferedReader reader = new BufferedReader(readerArg); + try { + fileSourceHost = reader.readLine(); + if (fileSourceHost == null) + throw new RuntimeException("Error while reading pre generated file registry"); + + String line; + while ((line = reader.readLine()) != null) { + addFromLine(line); + } + } catch(IOException e) { + throw new RuntimeException("Error while reading pre generated file registry", e); + } finally { + try { + reader.close(); + } catch(IOException e) {} + } + } + + private void addFromLine(String line) { + String[] parts = entryDelimiterPattern.split(line); + addEntry(parts[0], parts[1]); + } + + private void addEntry(String relativePath, String hash) { + path2Hash.put(relativePath, hash); + } + + public static String exportRegistry(FileRegistry registry) { + List<FileRegistry.Entry> entries = registry.export(); + StringBuilder builder = new StringBuilder(); + + builder.append(registry.fileSourceHost()).append('\n'); + for (FileRegistry.Entry entry : entries) { + builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()). + append('\n'); + } + + return builder.toString(); + } + + public static PreGeneratedFileRegistry importRegistry(Reader reader) { + return new PreGeneratedFileRegistry(reader); + } + + public FileReference addFile(String relativePath) { + return FileReferenceCreator.create(path2Hash.get(relativePath)); + } + + @Override + public String fileSourceHost() { + return fileSourceHost; + } + + public Set<String> getPaths() { + return path2Hash.keySet(); + } + + @Override + public Set<String> allRelativePaths() { + return path2Hash.keySet(); + } + + @Override + public List<Entry> export() { + List<Entry> entries = new ArrayList<>(); + for (Map.Entry<String, String> entry : path2Hash.entrySet()) { + entries.add(new Entry(entry.getKey(), FileReferenceCreator.create(entry.getValue()))); + } + return entries; + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java new file mode 100644 index 00000000000..a28a17dc831 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.thaiopensource.util.PropertyMap; +import com.thaiopensource.util.PropertyMapBuilder; +import com.thaiopensource.validate.ValidateProperty; +import com.thaiopensource.validate.ValidationDriver; +import com.thaiopensource.validate.rng.CompactSchemaReader; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; +import org.xml.sax.ErrorHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.Enumeration; +import java.util.Optional; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Validates xml files against one schema. + * + * @author tonytv + */ +public class SchemaValidator { + + public static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); + static final String servicesXmlSchemaName = "services.rnc"; + static final String hostsXmlSchemaName = "hosts.rnc"; + private final CustomErrorHandler errorHandler = new CustomErrorHandler(); + private final ValidationDriver driver; + private DeployLogger deployLogger; + private static final Logger log = Logger.getLogger(SchemaValidator.class.getName()); + + /** + * Initializes the validator by using the given file as schema file + * @param schema a schema file in RNC format + * @param logger a logger + */ + public SchemaValidator(String schema, DeployLogger logger, Optional<Version> vespaVersion) { + this.deployLogger = logger; + driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); + File schemaDir = new File(schemaDirBase); + try { + schemaDir = saveSchemasFromJar(new File(SchemaValidator.schemaDirBase), vespaVersion); + } catch (IOException e) { + throw new RuntimeException(e); + } + loadSchema(new File(schemaDir + File.separator + "schema" + File.separator + schema)); + IOUtils.recursiveDeleteDir(schemaDir); + } + + /** + * Initializes the validator by using the given file as schema file + * @param schema a schema file in RNC format + * @throws IOException if it is not possible to read schema files + */ + public SchemaValidator(String schema) throws IOException { + this(schema, new BaseDeployLogger(), Optional.empty()); + } + + /** + * Create a validator for services.xml for tests + * @throws IOException if it is not possible to read schema files + */ + public static SchemaValidator createTestValidatorServices() throws IOException { + return new SchemaValidator(servicesXmlSchemaName); + } + + /** + * Create a validator for hosts.xml for tests + * @throws IOException if it is not possible to read schema files + */ + public static SchemaValidator createTestValidatorHosts() throws IOException { + return new SchemaValidator(hostsXmlSchemaName); + } + + private class CustomErrorHandler implements ErrorHandler { + volatile String fileName; + + public void warning(SAXParseException e) throws SAXException { + deployLogger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + private String message(SAXParseException e) { + return "XML error in " + fileName + ": " + + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + } + + /** + * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. + * + * @return the directory the schema files are stored in + * @throws IOException if it is not possible to read schema files + */ + public File saveSchemasFromJar(File tmpBase, Optional<Version> vespaVersion) throws IOException { + final Class<? extends SchemaValidator> schemaValidatorClass = this.getClass(); + final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); + Enumeration<URL> uris = classLoader.getResources("schema"); + if (uris==null) return null; + File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); + log.log(LogLevel.DEBUG, "Saving schemas to " + tmpDir); + while(uris.hasMoreElements()) { + URL u = uris.nextElement(); + log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + if ("jar".equals(u.getProtocol())) { + JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Enumeration<JarEntry> entries = jarFile.entries(); + entries.hasMoreElements();) { + + JarEntry je=entries.nextElement(); + if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { + writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); + } + } + jarFile.close(); + } else if ("bundle".equals(u.getProtocol())) { + Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); + log.log(LogLevel.DEBUG, classLoader.toString()); + log.log(LogLevel.DEBUG, "bundle=" + bundle); + // TODO: Hack to handle cases where bundle=null + if (bundle == null) { + File schemaPath; + if (vespaVersion.isPresent() && vespaVersion.get().getMajor() == 5) { + schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/version/5.x/schema/"); + } else { + schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/"); + } + log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + copySchemas(schemaPath, tmpDir); + } else { + log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + .getVersion())); + for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); + entries.hasMoreElements(); ) { + + URL url = entries.nextElement(); + writeContentsToFile(tmpDir, url.getFile(), url.openStream()); + } + } + } else if ("file".equals(u.getProtocol())) { + File schemaPath = new File(u.getPath()); + copySchemas(schemaPath, tmpDir); + } + } + return tmpDir; + } + + private static void copySchemas(File from, File to) throws IOException { + // TODO: only copy .rnc files. + if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); + if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); + File sourceFile = new File(from, servicesXmlSchemaName); + if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); + IOUtils.copyDirectoryInto(from, to); + } + + private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { + String contents = IOUtils.readAll(new InputStreamReader(inputStream)); + File out = new File(outDir, outFile); + IOUtils.writeFile(out, contents, false); + } + + private void loadSchema(File schemaFile) { + try { + driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); + } catch (SAXException e) { + throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); + } catch (IOException e) { + throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); + } + } + + private PropertyMap instanceProperties() { + PropertyMapBuilder builder = new PropertyMapBuilder(); + builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); + return builder.toPropertyMap(); + } + + public void validate(File file) throws IOException { + validate(file, file.getName()); + } + + public void validate(File file, String fileName) throws IOException { + validate(ValidationDriver.fileInputSource(file), fileName); + } + + public void validate(Reader reader) throws IOException { + validate(new InputSource(reader), null); + } + + public void validate(NamedReader reader) throws IOException { + validate(new InputSource(reader), reader.getName()); + } + + public void validate(InputSource inputSource, String fileName) throws IOException { + errorHandler.fileName = (fileName == null ? " input" : fileName); + try { + if ( ! driver.validate(inputSource)) { + //Shouldn't happen, error handler should have thrown + throw new RuntimeException("Aborting due to earlier XML errors."); + } + } catch (SAXException e) { + //This should never happen, as it is handled by the ErrorHandler + //installed for the driver. + throw new IllegalArgumentException( + "XML error in " + (fileName == null ? " input" : fileName) + ": " + Exceptions.toMessageString(e)); + } + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java new file mode 100644 index 00000000000..80dc01be0b0 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import java.io.IOException; +import java.io.Reader; + +/** + * Simple Validation of services.xml for unit tests against RELAX NG schemas. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class SimpleApplicationValidator { + + public static void checkServices(Reader reader) throws IOException { + SchemaValidator.createTestValidatorServices().validate(reader); + } +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java new file mode 100644 index 00000000000..85ed84c6468 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.defaults.Defaults; +import org.apache.commons.lang.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A global pool of all config definitions that this server knows about. These objects can be shared + * by all tenants, as they are not modified. + * + * @author lulf + * @since 5.10 + */ +public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(StaticConfigDefinitionRepo.class.getName()); + private final Map<ConfigDefinitionKey, ConfigDefinition> configDefinitions = new LinkedHashMap<>(); + private static final String DEFAULT_SERVER_DEF_DIR = Defaults.getDefaults().vespaHome() + "var/db/vespa/config_server/serverdb/classes"; + + public StaticConfigDefinitionRepo() { + this(new File(DEFAULT_SERVER_DEF_DIR)); + } + + public StaticConfigDefinitionRepo(File definitionDir) { + initialize(definitionDir); + } + + private void initialize(File definitionDir) { + if ( ! definitionDir.exists()) return; + + for (File def : definitionDir.listFiles((dir, name) -> name.matches(".*\\.def"))) + addConfigDefinition(def); + } + + private void addConfigDefinition(File def) { + try { + ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); + if (key.getNamespace().isEmpty()) + key = new ConfigDefinitionKey(key.getName(), CNode.DEFAULT_NAMESPACE); + addConfigDefinition(key, def); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Exception adding config definition " + def, e); + } + } + + private void addConfigDefinition(ConfigDefinitionKey key, File defFile) throws IOException { + String payload = IOUtils.readFile(defFile); + configDefinitions.put(key, new ConfigDefinition(key.getName(), StringUtils.split(payload, "\n"))); + } + + @Override + public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { + return Collections.unmodifiableMap(configDefinitions); + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java new file mode 100644 index 00000000000..48e80d70312 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.application.provider; + +import com.yahoo.osgi.annotation.ExportPackage; |