diff options
Diffstat (limited to 'config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java')
| -rw-r--r-- | config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java | 772 |
1 files changed, 772 insertions, 0 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java new file mode 100644 index 00000000000..21e7de8fa7b --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -0,0 +1,772 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.config.application.ConfigDefinitionDir; +import com.yahoo.config.application.Xml; +import com.yahoo.config.application.XmlPreProcessor; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.RuleConfigDeriver; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.path.Path; +import com.yahoo.io.HexDump; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.util.ConfigUtils; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.*; +import java.net.JarURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.security.MessageDigest; +import java.util.*; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.io.IOUtils.readAll; +import static com.yahoo.text.Lowercase.toLowerCase; + + +/** + * Application package derived from local files, ie. during deploy. + * Construct using {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFile(java.io.File)} or + * {@link com.yahoo.config.model.application.provider.FilesApplicationPackage#fromFileWithDeployData(java.io.File, DeployData)}. + * + * @author vegardh + */ +public class FilesApplicationPackage implements ApplicationPackage { + + private static final Logger log = Logger.getLogger(FilesApplicationPackage.class.getName()); + private static final String META_FILE_NAME = ".applicationMetaData"; + + private final File appDir; + private final File preprocessedDir; + private final File configDefsDir; + private final AppSubDirs appSubDirs; + // NOTE: these directories exist in the original user app, but their locations are given in 'services.xml' + private final List<String> userIncludeDirs = new ArrayList<>(); + private final ApplicationMetaData metaData; + + /** + * Returns an application package object based on the given application dir + * + * @param appDir application package directory + * @return an Application package instance + */ + public static FilesApplicationPackage fromFile(File appDir) { + return new Builder(appDir).preprocessedDir(new File(appDir, ".preprocessed")).build(); + } + + /** Creates package from a local directory, typically deploy app */ + public static FilesApplicationPackage fromFileWithDeployData(File appDir, DeployData deployData) { + return new Builder(appDir).deployData(deployData).build(); + } + + /** + * Builder for {@link com.yahoo.config.model.application.provider.FilesApplicationPackage}. Use + * this to create instances in a flexible manner. + */ + public static class Builder { + private final File appDir; + private Optional<File> preprocessedDir = Optional.empty(); + private Optional<ApplicationMetaData> metaData = Optional.empty(); + + public Builder(File appDir) { + this.appDir = appDir; + } + + public Builder preprocessedDir(File preprocessedDir) { + this.preprocessedDir = Optional.ofNullable(preprocessedDir); + return this; + } + + public Builder deployData(DeployData deployData) { + this.metaData = Optional.of(metaDataFromDeployData(appDir, deployData)); + return this; + } + + public FilesApplicationPackage build() { + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, ".preprocessed")), metaData.orElse(readMetaData(appDir))); + } + } + + private static ApplicationMetaData metaDataFromDeployData(File appDir, DeployData deployData) { + return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), deployData.getDeployTimestamp(), + deployData.getApplicationName(), computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); + } + + /** + * New package from given path on local file system. Retrieves config definition files from + * the default location 'serverdb/classes'. + * + * @param appDir application package directory + * @param preprocessedDir preprocessed application package output directory + * @param metaData metadata for this application package + */ + @SuppressWarnings("deprecation") + private FilesApplicationPackage(File appDir, File preprocessedDir, ApplicationMetaData metaData) { + verifyAppDir(appDir); + this.appDir = appDir; + this.preprocessedDir = preprocessedDir; + appSubDirs = new AppSubDirs(appDir); + configDefsDir = new File(appDir, ApplicationPackage.CONFIG_DEFINITIONS_DIR); + addUserIncludeDirs(); + this.metaData = metaData; + } + + public String getApplicationName() { + return metaData.getApplicationName(); + } + + @Override + public List<NamedReader> getFiles(Path relativePath, String suffix, boolean recurse) { + return getFiles(relativePath, "", suffix, recurse); + } + + @Override + public ApplicationFile getFile(Path path) { + File file = (path.isRoot() ? appDir : new File(appDir, path.getRelative())); + return new FilesApplicationFile(path, file); + } + + @Override + public ApplicationMetaData getMetaData() { + return metaData; + } + + private List<NamedReader> getFiles(Path relativePath,String namePrefix,String suffix,boolean recurse) { + try { + List<NamedReader> readers=new ArrayList<>(); + File dir = new File(appDir, relativePath.getRelative()); + if ( ! dir.isDirectory()) return readers; + + final File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + if (recurse) + readers.addAll(getFiles(relativePath.append(file.getName()), namePrefix + "/" + file.getName(), suffix, recurse)); + } else { + if (suffix == null || file.getName().endsWith(suffix)) + readers.add(new NamedReader(file.getName(), new FileReader(file))); + } + } + } + return readers; + } + catch (IOException e) { + throw new RuntimeException("Could not open (all) files in '" + relativePath + "'",e); + } + } + + private void verifyAppDir(File appDir) { + if (appDir==null || !appDir.isDirectory()) { + throw new IllegalArgumentException("Path '" + appDir + "' is not a directory."); + } + if (! appDir.canRead()){ + throw new IllegalArgumentException("Cannot read from application directory '" + appDir + "'"); + } + } + + @Override + public Reader getHosts() { + try { + File hostsFile = getHostsFile(); + if (!hostsFile.exists()) return null; + return new FileReader(hostsFile); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public String getHostSource() { + return getHostsFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getHostsFile() { + return new File(appDir, ApplicationPackage.HOSTS); + } + + private File getFileWithFallback(String first, String second) { + File firstFile = new File(appDir, first); + File secondFile = new File(appDir, second); + if (firstFile.exists()) { + return firstFile; + } else if (secondFile.exists()) { + return secondFile; + } else { + return firstFile; + } + } + + @Override + public String getServicesSource() { + return getServicesFile().getPath(); + } + + @SuppressWarnings("deprecation") + private File getServicesFile() { + return new File(appDir, ApplicationPackage.SERVICES); + } + + @Override + public Optional<Reader> getDeployment() { return optionalFile(DEPLOYMENT_FILE); } + + @Override + public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES); } + + private Optional<Reader> optionalFile(Path filePath) { + try { + return Optional.of(getFile(filePath).createReader()); + } catch (FileNotFoundException e) { + return Optional.empty(); + } + } + + + @Override + public List<String> getUserIncludeDirs() { + return Collections.unmodifiableList(userIncludeDirs); + } + + public void addUserIncludeDirs() { + Document services; + try { + services = Xml.getDocument(getServices()); + } catch (Exception e) { + return; // This method does not validate that services.xml exists, or that it is valid xml. + } + NodeList includeNodes = services.getElementsByTagName(IncludeDirs.INCLUDE); + + for (int i=0; i < includeNodes.getLength(); i++) { + Node includeNode = includeNodes.item(i); + addIncludeDir(includeNode); + } + } + + private void addIncludeDir(Node includeNode) { + if (! (includeNode instanceof Element)) + return; + Element include = (Element) includeNode; + if (! include.hasAttribute(IncludeDirs.DIR)) + return; + String dir = include.getAttribute(IncludeDirs.DIR); + validateIncludeDir(dir); + IncludeDirs.validateFilesInIncludedDir(dir, include.getParentNode(), this); + log.log(LogLevel.INFO, "Adding user include dir '" + dir + "'"); + userIncludeDirs.add(dir); + } + + @Override + public void validateIncludeDir(String dirName) { + IncludeDirs.validateIncludeDir(dirName, this); + } + + @Override + public Collection<NamedReader> searchDefinitionContents() { + Map<String, NamedReader> ret = new LinkedHashMap<>(); + Set<String> fileSds = new LinkedHashSet<>(); + Set<String> bundleSds = new LinkedHashSet<>(); + try { + for (File f : getSearchDefinitionFiles()) { + fileSds.add(f.getName()); + ret.put(f.getName(), new NamedReader(f.getName(), new FileReader(f))); + } + for (Map.Entry<String, String> e : allSdsFromDocprocBundlesAndClasspath(appDir).entrySet()) { + bundleSds.add(e.getKey()); + ret.put(e.getKey(), new NamedReader(e.getKey(), new StringReader(e.getValue()))); + } + } catch (Exception e) { + throw new IllegalArgumentException("Couldn't get search definition contents.", e); + } + verifySdsDisjoint(fileSds, bundleSds); + return ret.values(); + } + + /** + * Verify that two sets of search definitions are disjoint (TODO: everything except error message is very generic). + * @param fileSds Set of search definitions from file + * @param bundleSds Set of search definitions from bundles + */ + private void verifySdsDisjoint(Set<String> fileSds, Set<String> bundleSds) { + if (!Collections.disjoint(fileSds, bundleSds)) { + Collection<String> disjoint = new ArrayList<>(fileSds); + disjoint.retainAll(bundleSds); + throw new IllegalArgumentException("For the following search definitions names there are collisions between those specified inside " + + "docproc bundles and those in searchdefinitions/ in application package: "+disjoint); + } + } + + /** + * Returns sdName→payload for all SDs in all docproc bundles and on local classpath. + * Throws {@link IllegalArgumentException} if there are multiple sd files of same name. + * @param appDir application package directory + * @return a map from search definition name to search definition content + * @throws IOException if reading a search definition fails + */ + public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { + File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); + if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); + List<String> usedNames = new ArrayList<>(); + Map<String, String> ret = new LinkedHashMap<>(); + + // try classpath first + allSdsOnClassPath(usedNames, ret); + + for (File bundle : dpChains.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + }})) { + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+ + bundle.getName()+"' is already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + return ret; + } + + private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException { + Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + + while(resources.hasMoreElements()) { + URL resource = resources.nextElement(); + + String protocol = resource.getProtocol(); + + if ("file".equals(protocol)) { + File file; + try { + file = new File(resource.toURI()); + } catch (URISyntaxException e) { + continue; + } + // only interested in directories + if (file.isDirectory()) { + List<File> sdFiles = getSearchDefinitionFiles(file); + for (File sdFile : sdFiles) { + String sdName = sdFile.getName(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '"+sdName+ + "' found in classpath already used earlier in classpath."); + } + usedNames.add(sdName); + String contents = IOUtils.readAll(new FileReader(sdFile)); + ret.put(sdFile.getName(), contents); + } + } + } + else if ("jar".equals(protocol)) { + JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { + String sdName = entry.getKey(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definitions name '"+sdName+ + "' used in bundle '"+jarFile.getName()+"' already used in classpath or previous bundle."); + } + usedNames.add(sdName); + String sdPayload = entry.getValue(); + ret.put(sdName, sdPayload); + } + } + } + } + + private Reader retrieveConfigDefReader(String defName) { + File def = new File(configDefsDir + File.separator + defName); + try { + return new NamedReader(def.getAbsolutePath(), new FileReader(def)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read config definition file '" + + def.getAbsolutePath() + "'", e); + } + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs = new LinkedHashMap<>(); + + if (configDefsDir.isDirectory()) { + addAllDefsFromConfigDir(defs, configDefsDir); + } + addAllDefsFromBundles(defs, FilesApplicationPackage.getComponents(appDir)); + return defs; + } + + private void addAllDefsFromBundles(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, List<Component> components) { + for (Component component : components) { + Bundle bundle = component.getBundle(); + for (final Bundle.DefEntry def : bundle.getDefEntries()) { + final ConfigDefinitionKey defKey = new ConfigDefinitionKey(def.defName, def.defNamespace); + if (!defs.containsKey(defKey)) { + defs.put(defKey, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(defKey.getName(), new StringReader(def.contents)); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return def.contents; + } + }); + } + } + } + } + + private void addAllDefsFromConfigDir(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, File configDefsDir) { + log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); + for (final File def : configDefsDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\.def");}})) { + + log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'"); + String[] nv = def.getName().split("\\.def"); + if (nv == null) { + log.log(LogLevel.WARNING, "Skipping '" + def + "', cannot determine name"); + } else { + ConfigDefinitionKey key; + try { + key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); + } catch (IOException e) { + e.printStackTrace(); + break; + } + if (key.getNamespace().isEmpty()) { + throw new IllegalArgumentException("Config definition '" + nv + "' has no namespace"); + } + boolean addFile = false; + if (defs.containsKey(key)) { + if (nv[0].contains(".")) { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". The file '" + def + "' will take precedence"); + addFile = true; + } else { + log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + ". Skipping '" + def + "', as it does not contain namespace in filename"); + } + } else { + addFile = true; + } + if (addFile) { + log.log(LogLevel.DEBUG, "Adding " + key + " to archive of all existing config defs"); + final ConfigDefinitionKey finalKey = key; + defs.put(key, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(finalKey.getName(), retrieveConfigDefReader(def.getName())); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + return readConfigDefinition(def.getName()); + } + }); + } + } + } + } + + private String readConfigDefinition(String name) { + try (Reader reader = retrieveConfigDefReader(name)) { + return IOUtils.readAll(reader); + } catch (IOException e) { + throw new RuntimeException("Error reading config definition " + name, e); + } + } + + @Override + public Reader getServices() { + try { + return new FileReader(getServicesSource()); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + //Only intended for DeployProcessor, others should use the member version + static List<File> getSearchDefinitionFiles(File appDir) { + //The dot is escaped later in this method: + assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.'); + + List<File> ret = new ArrayList<>(); + File sdDir; + + sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + if (!sdDir.isDirectory()) { + return ret; + } + ret.addAll(Arrays.asList( + sdDir.listFiles( + new FilenameFilter() { @Override public boolean accept(File dir, String name) { + return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}}))); + return ret; + } + + public List<File> getSearchDefinitionFiles() { + return getSearchDefinitionFiles(appDir); + } + + //Only for use by deploy processor + public static List<Component> getComponents(File appDir) { + List<Component> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new Component(bundle, new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath()))); + } + return components; + } + + private static List<ComponentInfo> getComponentsInfo(File appDir) { + List<ComponentInfo> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { + components.add(new ComponentInfo(new File(ApplicationPackage.COMPONENT_DIR, bundle.getFile().getName()).getPath())); + } + return components; + } + + @Override + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + return getComponentsInfo(appDir); + } + + /** + * Returns a list of all components in this package. + * + * @return A list of components. + */ + public List<Component> getComponents() { + return getComponents(appDir); + } + + public File getAppDir() throws IOException { + return appDir.getCanonicalFile(); + } + + public static ApplicationMetaData readMetaData(File appDir) { + ApplicationMetaData defaultMetaData = new ApplicationMetaData(appDir, "n/a", "n/a", 0l, "", 0l, 0l); + File metaFile = new File(appDir, META_FILE_NAME); + if (!metaFile.exists()) { + return defaultMetaData; + } + try (FileReader reader = new FileReader(metaFile)) { + return ApplicationMetaData.fromJsonString(IOUtils.readAll(reader)); + } catch (Exception e) { + // Not a big deal, return default + return defaultMetaData; + } + } + + /** + * Represents a component in the application package. Immutable. + */ + public static class Component { + + public final ComponentInfo info; + private final Bundle bundle; + + public Component(Bundle bundle, ComponentInfo info) { + this.bundle = bundle; + this.info = info; + } + + public List<Bundle.DefEntry> getDefEntries() { + return bundle.getDefEntries(); + } + + public Bundle getBundle() { + return bundle; + } + } // class Component + + /** + * Reads a ranking expression from file to a string and returns it. + * + * @param name the name of the file to return, either absolute or + * relative to the search definition directory in the application package + * @return the content of a ranking expression file + * @throws IllegalArgumentException if the file was not found or could not be read + */ + // TODO: A note on absolute paths: We don't want to support this and it should be removed on 6.0 + // Currently one system test (basicmlr) depends on it. + @Override + public Reader getRankingExpression(String name) { + try { + return IOUtils.createReader(expressionFileNameToFile(name), "utf-8"); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read ranking expression file '" + name + "'", e); + } + } + + private File expressionFileNameToFile(String name) { + File expressionFile = new File(name); + if (expressionFile.isAbsolute()) return expressionFile; + + File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + return new File(sdDir, name); + } + + @Override + public File getFileReference(Path pathRelativeToAppDir) { + return new File(appDir, pathRelativeToAppDir.getRelative()); + } + + @Override + public void validateXML(DeployLogger logger) throws IOException { + validateXML(logger, Optional.empty()); + } + + @Override + public void validateXML(DeployLogger logger, Optional<Version> vespaVersion) throws IOException { + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, logger, vespaVersion); + xmlFilesValidator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); + } + + @Override + public void writeMetaData() throws IOException { + File metaFile = new File(appDir, META_FILE_NAME); + IOUtils.writeFile(metaFile, metaData.asJsonString(), false); + } + + @Override + public Collection<NamedReader> getSearchDefinitions() { + return searchDefinitionContents(); + } + + private void preprocessXML(File destination, File inputXml, Zone zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { + Document document = new XmlPreProcessor(appDir, inputXml, zone.environment(), zone.region()).run(); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + try (FileOutputStream outputStream = new FileOutputStream(destination)) { + transformer.transform(new DOMSource(document), new StreamResult(outputStream)); + } + } + + @Override + public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { + IOUtils.recursiveDeleteDir(preprocessedDir); + IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> !name.equals(".preprocessed") && + !name.equals(SERVICES) && + !name.equals(HOSTS) && + !name.equals(CONFIG_DEFINITIONS_DIR)); + preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); + if (getHostsFile().exists()) { + preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); + } + FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir); + preprocessed.copyUserDefsIntoApplication(); + return preprocessed; + } + + private void copyUserDefsIntoApplication() { + File destination = appSubDirs.configDefs(); + destination.mkdir(); + ConfigDefinitionDir defDir = new ConfigDefinitionDir(destination); + // Copy the user's def files from components. + List<Bundle> bundlesAdded = new ArrayList<>(); + for (FilesApplicationPackage.Component component : FilesApplicationPackage.getComponents(appSubDirs.root())) { + Bundle bundle = component.getBundle(); + defDir.addConfigDefinitionsFromBundle(bundle, bundlesAdded); + bundlesAdded.add(bundle); + } + } + + /** + * Computes an md5 hash of the contents of the application package + * + * @return an md5sum of the application package + */ + private static String computeCheckSum(File appDir) { + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + for (File file : appDir.listFiles((dir, name) -> !name.equals(ApplicationPackage.EXT_DIR) && !name.startsWith("."))) { + addPathToDigest(file, "", md5, true, false); + } + return toLowerCase(HexDump.toHexString(md5.digest())); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Adds the given path to the digest, or does nothing if path is neither file nor dir + * @param path path to add to message digest + * @param suffix only files with this suffix are considered + * @param digest the {link @MessageDigest} to add the file paths to + * @param recursive whether to recursively find children in the paths + * @param fullPathNames Whether to include the full paths in checksum or only the names + * @throws java.io.IOException if adding path to digest fails when reading files from path + */ + private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException { + if (!path.exists()) return; + if (fullPathNames) { + digest.update(path.getPath().getBytes(Utf8.getCharset())); + } else { + digest.update(path.getName().getBytes(Utf8.getCharset())); + } + if (path.isFile()) { + FileInputStream is = new FileInputStream(path); + addToDigest(is, digest); + is.close(); + } else if (path.isDirectory()) { + final File[] files = path.listFiles(); + if (files != null) { + for (File elem : files) { + if ((elem.isDirectory() && recursive) || elem.getName().endsWith(suffix)) { + addPathToDigest(elem, suffix, digest, recursive, fullPathNames); + } + } + } + } + } + + private static final int MD5_BUFFER_SIZE = 65536; + private static void addToDigest(InputStream is, MessageDigest digest) throws IOException { + if (is==null) return; + byte[] buffer = new byte[MD5_BUFFER_SIZE]; + int i; + do { + i=is.read(buffer); + if (i > 0) { + digest.update(buffer, 0, i); + } + } while(i!=-1); + } + +} |