diff options
11 files changed, 31 insertions, 247 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 2e4341f59ae..5f247afae03 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -6,29 +6,27 @@ import com.yahoo.component.Vtag; import com.yahoo.config.application.ConfigDefinitionDir; import com.yahoo.config.application.Xml; import com.yahoo.config.application.XmlPreProcessor; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.path.Path; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; -import java.util.logging.Level; +import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.util.ConfigUtils; - import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -41,15 +39,28 @@ import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import java.io.*; -import java.net.JarURLConnection; -import java.net.URISyntaxException; -import java.net.URL; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; import java.security.MessageDigest; -import java.util.*; -import java.util.jar.JarFile; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Stream; import static com.yahoo.text.Lowercase.toLowerCase; @@ -301,10 +312,6 @@ public class FilesApplicationPackage implements ApplicationPackage { fileSds.add(f.getName()); ret.put(f.getName(), new NamedReader(f.getName(), new FileReader(f))); } - for (Map.Entry<String, String> e : allSdsFromDocprocBundlesAndClasspath(appDir).entrySet()) { - bundleSds.add(e.getKey()); - ret.put(e.getKey(), new NamedReader(e.getKey(), new StringReader(e.getValue()))); - } } catch (Exception e) { throw new IllegalArgumentException("Couldn't get search definition contents.", e); } @@ -328,85 +335,6 @@ public class FilesApplicationPackage implements ApplicationPackage { } /** - * Returns sdName→payload for all SDs in all docproc bundles and on local classpath. - * Throws {@link IllegalArgumentException} if there are multiple sd files of same name. - * @param appDir application package directory - * @return a map from search definition name to search definition content - * @throws IOException if reading a search definition fails - */ - public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { - File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); - if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); - Set<String> usedNames = new HashSet<>(); - Map<String, String> schemas = new LinkedHashMap<>(); - - // try classpath first - allSdsOnClassPath(usedNames, schemas); - - for (File bundle : dpChains.listFiles((File dir, String name) -> name.endsWith(".jar"))) { - for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { - String sdName = entry.getKey(); - if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '" + sdName + "' used in bundle '"+ - bundle.getName()+ "' is already used in classpath or previous bundle."); - } - usedNames.add(sdName); - String sdPayload = entry.getValue(); - schemas.put(sdName, sdPayload); - } - } - return schemas; - } - - private static void allSdsOnClassPath(Set<String> usedNames, Map<String, String> schemas) { - ClassLoader cl = FilesApplicationPackage.class.getClassLoader(); - Stream<URL> resources = Stream.concat(cl.resources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()), - cl.resources(ApplicationPackage.SCHEMAS_DIR.getRelative())); - resources.forEach(resource -> addSchemaFrom(resource, schemas, usedNames)); - } - - private static void addSchemaFrom(URL resource, Map<String, String> schemas, Set<String> usedNames) { - try { - switch (resource.getProtocol()) { - case "file": - File file = new File(resource.toURI()); - if (file.isDirectory()) { - List<File> sdFiles = getSearchDefinitionFiles(file); - for (File sdFile : sdFiles) { - String sdName = sdFile.getName(); - if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '" + sdName + - "' found in classpath already used earlier in classpath."); - } - usedNames.add(sdName); - String contents = IOUtils.readAll(new FileReader(sdFile)); - schemas.put(sdFile.getName(), contents); - } - } - break; - case "jar": - JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); - JarFile jarFile = jarConnection.getJarFile(); - for (Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { - String sdName = entry.getKey(); - if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definitions name '" + sdName + - "' used in bundle '" + jarFile.getName() + "' " + - "is already used in classpath or previous bundle."); - } - usedNames.add(sdName); - String sdPayload = entry.getValue(); - schemas.put(sdName, sdPayload); - } - break; - } - } - catch (IOException | URISyntaxException e) { - throw new IllegalArgumentException("Could not read schema from '" + resource + "'", e); - } - } - - /** * Creates a reader for a config definition * * @param defPath the path to the application package diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 174f9bb54d7..cdab2dc2fb2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.component.Version; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.XML; @@ -17,12 +16,9 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; import java.io.Reader; import java.util.Collection; import java.util.Collections; -import java.util.Enumeration; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -204,27 +200,9 @@ public interface ApplicationPackage { */ Reader getRankingExpression(String name); - /** - * Returns the name-payload pairs of any sd files under path/schemas and path/searchdefinitions/ - * in the given jar bundle. - * - * @param bundle the jar file, which will be closed afterwards by this method - * @param path for example 'complex/' - * @return map of the SD payloads - * @throws IOException if it is reading sd files fails - */ + // TODO: Unused, remove when 7.264 is latest version in use static Map<String, String> getBundleSdFiles(String path, JarFile bundle) throws IOException { - Map<String, String> schemas = new LinkedHashMap<>(); - for (Enumeration<JarEntry> e = bundle.entries(); e.hasMoreElements();) { - JarEntry entry = e.nextElement(); - if ((entry.getName().startsWith(path + SCHEMAS_DIR + "/") || entry.getName().startsWith(path + SEARCH_DEFINITIONS_DIR + "/")) - && entry.getName().endsWith(SD_NAME_SUFFIX)) { - String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(entry))); - schemas.put(getFileName(entry), contents); - } - } - bundle.close(); - return schemas; + return Map.of(); } /** diff --git a/config-model/src/test/cfg/application/app_sdbundles/components/testbundle.jar b/config-model/src/test/cfg/application/app_sdbundles/components/testbundle.jar Binary files differdeleted file mode 100644 index 00749d776c2..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/components/testbundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/app_sdbundles/components/testbundle2.jar b/config-model/src/test/cfg/application/app_sdbundles/components/testbundle2.jar Binary files differdeleted file mode 100644 index 36c97c2716c..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/components/testbundle2.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/app_sdbundles/files/foo.txt b/config-model/src/test/cfg/application/app_sdbundles/files/foo.txt deleted file mode 100644 index b7d6715e2df..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/files/foo.txt +++ /dev/null @@ -1 +0,0 @@ -FOO diff --git a/config-model/src/test/cfg/application/app_sdbundles/files/subdir/bar.txt b/config-model/src/test/cfg/application/app_sdbundles/files/subdir/bar.txt deleted file mode 100644 index ba578e48b18..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/files/subdir/bar.txt +++ /dev/null @@ -1 +0,0 @@ -BAR diff --git a/config-model/src/test/cfg/application/app_sdbundles/hosts.xml b/config-model/src/test/cfg/application/app_sdbundles/hosts.xml deleted file mode 100644 index 9cbdd000786..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/hosts.xml +++ /dev/null @@ -1,11 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> - <host name="schmocalhost"> - <alias>node2</alias> - </host> -</hosts> - diff --git a/config-model/src/test/cfg/application/app_sdbundles/services.xml b/config-model/src/test/cfg/application/app_sdbundles/services.xml deleted file mode 100644 index 6b59d13844a..00000000000 --- a/config-model/src/test/cfg/application/app_sdbundles/services.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - <slobroks> - <slobrok hostalias="node1"/> - <slobrok hostalias="node2"/> - </slobroks> - </admin> - - <container version="1.0"> - <nodes> - <node hostalias="node1" /> - </nodes> - - <search/> - <document-api/> - </container> - - <content id="music" version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index" /> - </documents> - <nodes> - <node hostalias="node1" distribution-key="0" /> - </nodes> - </content> - -</services> diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index fe82f2406f2..2c209dd79bf 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -2,21 +2,19 @@ package com.yahoo.config.model; import com.google.common.io.Files; -import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.model.application.provider.Bundle; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.path.Path; import com.yahoo.document.DataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.Search; +import com.yahoo.path.Path; import com.yahoo.searchdefinition.DocumentOnlySearch; +import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; @@ -38,8 +36,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -134,28 +130,6 @@ public class ApplicationDeployTest { } @Test - public void testSdFromDocprocBundle() throws IOException, SAXException { - String appDir = "src/test/cfg/application/app_sdbundles"; - ApplicationPackageTester tester = ApplicationPackageTester.create(appDir); - VespaModel model = new VespaModel(tester.app()); - DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); - model.getConfig(b, VespaModel.ROOT_CONFIGID); - DocumentmanagerConfig dc = b.build(); - String docMan=ConfigInstance.serialize(dc).toString(); - int pFlags = Pattern.MULTILINE + Pattern.DOTALL; - Pattern base = Pattern.compile(".*name.*base\\.header.*", pFlags); - Pattern book = Pattern.compile(".*name.*book\\.header.*", pFlags); - Pattern music = Pattern.compile(".*name.*music\\.header.*", pFlags); - Pattern video = Pattern.compile(".*name.*video\\.header.*", pFlags); - Pattern muzak = Pattern.compile(".*name.*muzak\\.header.*", pFlags); - assertTrue(base.matcher(docMan).matches()); - assertTrue(book.matcher(docMan).matches()); - assertTrue(music.matcher(docMan).matches()); - assertTrue(video.matcher(docMan).matches()); - assertTrue(muzak.matcher(docMan).matches()); - } - - @Test public void include_dirs_are_included() { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "include_dirs"); @@ -234,22 +208,6 @@ public class ApplicationDeployTest { } @Test - public void testGetJars() throws IOException { - String jarName = "src/test/cfg/application/app_sdbundles/components/testbundle.jar"; - JarFile jar = new JarFile(jarName); - Map<String, String> payloads = ApplicationPackage.getBundleSdFiles("", jar); - assertEquals(payloads.size(), 4); - assertTrue(payloads.get("base.sd").startsWith("search base")); - assertTrue(payloads.get("book.sd").startsWith("search book")); - assertTrue(payloads.get("music.sd").startsWith("search music")); - assertTrue(payloads.get("video.sd").startsWith("search video")); - assertTrue(payloads.get("base.sd").endsWith("}")); - assertTrue(payloads.get("book.sd").endsWith("}\n")); - assertTrue(payloads.get("music.sd").endsWith("}\n")); - assertTrue(payloads.get("video.sd").endsWith("}\n")); - } - - @Test public void testConfigDefinitionsFromJars() { String appName = "src/test/cfg//application/app1"; FilesApplicationPackage app = FilesApplicationPackage.fromFile(new File(appName), false); diff --git a/configserver/src/test/apps/zkfeed/components/defs-only.jar b/configserver/src/test/apps/zkfeed/components/defs-only.jar Binary files differdeleted file mode 100644 index 28f563cd779..00000000000 --- a/configserver/src/test/apps/zkfeed/components/defs-only.jar +++ /dev/null diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index a4fce5e37ba..5105f1b0d5f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -100,38 +99,18 @@ public class ZooKeeperClientTest { String defsPath = appPath + ConfigCurator.DEFCONFIGS_ZK_SUBPATH; assertTrue(zk.exists(appPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); List<String> children = zk.getChildren(defsPath); - assertEquals(defsPath + " children", 2, children.size()); + assertEquals(defsPath + " children", 1, children.size()); Collections.sort(children); assertThat(children.get(0), is("a.b.test2")); assertTrue(zk.exists(appPath, ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); String userDefsPath = appPath + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; children = zk.getChildren(userDefsPath); - assertThat(children.size(), is(2)); + assertThat(children.size(), is(1)); Collections.sort(children); assertThat(children.get(0), is("a.b.test2")); } - // TODO: Evaluate if we want this or not - @Test - @Ignore - public void testFeedComponentsFileReferencesToZooKeeper() { - final String appDir = "src/test/apps/app_sdbundles"; - ConfigCurator zk = ConfigCurator.create(new MockCurator()); - BaseDeployLogger logger = new BaseDeployLogger(); - Path app = Path.fromString("/1"); - ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app); - zooKeeperClient.setupZooKeeper(); - - String currentAppPath = app.getAbsolute(); - assertTrue(zk.exists(currentAppPath, ConfigCurator.USERAPP_ZK_SUBPATH.replaceFirst("/", ""))); - assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH, "components")); - assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle.jar")); - assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle2.jar")); - String data = zk.getData(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle2.jar"); - assertThat(data, is(new File(appDir + "/components/testbundle2.jar").getAbsolutePath())); - } - @Test public void testFeedAppMetaDataToZooKeeper() { assertTrue(zk.exists(appPath, ConfigCurator.META_ZK_PATH)); @@ -168,20 +147,6 @@ public class ZooKeeperClientTest { assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml").getAbsolute())); } - @Test - public void search_definitions_written_to_ZK() { - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("music.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("base.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("video.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("book.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("pc.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("laptop.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("product.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("sock.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("foo.expression").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("bar.expression").getAbsolute())); - } - private Path appPath() { return Path.fromString(appPath).append(ConfigCurator.USERAPP_ZK_SUBPATH); } |