From 79f4cb5be4a6719d0b11cd39c3f44af2849903b8 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 17 Mar 2020 11:41:59 +0100 Subject: Revert "Merge pull request #12565 from vespa-engine/revert-12563-bratseth/search-to-schema" This reverts commit ea5fe1bf8c0d389a3964d2e5c490994920d21a4e, reversing changes made to 7e92c3852e893d09f0aa821e04e4e9dbe83b4ab2. --- .../config/server/deploy/ZooKeeperClient.java | 84 +++++++++------------- .../config/server/zookeeper/ZKApplication.java | 13 ++-- .../server/zookeeper/ZKApplicationPackage.java | 22 +++--- .../config/server/deploy/ZooKeeperClientTest.java | 20 +++--- .../server/zookeeper/ZKApplicationPackageTest.java | 2 +- 5 files changed, 64 insertions(+), 77 deletions(-) (limited to 'configserver/src') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 13ef19f5f5d..9cb97b9cfb5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -87,17 +87,16 @@ public class ZooKeeperClient { /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */ private void createZooKeeperNodes() { - if (!configCurator.exists(rootPath.getAbsolute())) { + if ( ! configCurator.exists(rootPath.getAbsolute())) configCurator.createNode(rootPath.getAbsolute()); - } - for (String subPath : Arrays.asList( - ConfigCurator.DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USERAPP_ZK_SUBPATH, - ZKApplicationPackage.fileRegistryNode)) { - // TODO The replaceFirst below is hackish. - configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", "")); + for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USERAPP_ZK_SUBPATH, + ZKApplicationPackage.fileRegistryNode)) { + // TODO: The replaceFirst below is hackish. + configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), + subPath.replaceFirst("/", "")); } } @@ -108,7 +107,6 @@ public class ZooKeeperClient { */ void write(ApplicationPackage app) { logFine("Feeding application config into ZooKeeper"); - // gives lots and lots of debug output: // BasicConfigurator.configure(); try { logFine("Feeding user def files into ZooKeeper"); writeUserDefs(app); @@ -121,43 +119,33 @@ public class ZooKeeperClient { write(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + - "Please ensure that cloudconfig_server is started on the config server node(s), " + - "and check the vespa log for configserver errors. ", e); + "Please ensure that cloudconfig_server is started on the config server node(s), " + + "and check the vespa log for configserver errors. ", e); } } private void writeSearchDefinitions(ApplicationPackage app) throws IOException { Collection sds = app.getSearchDefinitions(); - if (sds.isEmpty()) { - return; - } - Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + if (sds.isEmpty()) return; + + Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SCHEMAS_DIR); configCurator.createNode(zkPath.getAbsolute()); - // Ensures that ranking expressions and other files are also fed. + // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); for (NamedReader sd : sds) { - String name = sd.getName(); - Reader reader = sd.getReader(); - String data = com.yahoo.io.IOUtils.readAll(reader); - reader.close(); - configCurator.putData(zkPath.getAbsolute(), name, data); + configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader())); + sd.getReader().close(); } } /** * Puts some of the application package files into ZK - see write(app). * - * @param app The application package to use as input. - * @throws java.io.IOException if not able to write to Zookeeper + * @param app the application package to use as input. + * @throws java.io.IOException if not able to write to Zookeeper */ private void writeSomeOf(ApplicationPackage app) throws IOException { - ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX); - } - }; - // Copy app package files and subdirs into zk // TODO: We should have a way of doing this which doesn't require repeating all the content writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); @@ -169,7 +157,8 @@ public class ZooKeeperClient { getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); writeDir(app.getFile(ApplicationPackage.RULES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), - srFilter, true); + (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX), + true); writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), xmlFilter, true); @@ -194,20 +183,12 @@ public class ZooKeeperClient { } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return true; - } - }, recurse); + writeDir(file, zooKeeperAppPath, (__) -> true, recurse); } private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { - if (!dir.isDirectory()) { - logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (ApplicationFile file: listFiles(dir, filenameFilter)) { + if ( ! dir.isDirectory()) return; + for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... if ("CVS".equals(name)) continue; @@ -223,7 +204,8 @@ public class ZooKeeperClient { } /** - * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories. + * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} + * with slightly different semantics: Never filter out directories. */ private List listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) { List rawList = dir.listFiles(); @@ -243,9 +225,8 @@ public class ZooKeeperClient { } private void writeFile(ApplicationFile file, Path zkPath) throws IOException { - if (!file.exists()) { - return; - } + if ( ! file.exists()) return; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream inputStream = file.createInputStream()) { inputStream.transferTo(baos); @@ -292,8 +273,8 @@ public class ZooKeeperClient { String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), - vespaVersion.toFullString(), - exportedRegistry); + vespaVersion.toFullString(), + exportedRegistry); } /** @@ -343,9 +324,8 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData( - rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), - AllocatedHostsSerializer.toJson(hosts)); + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 3ae678969eb..5f180bdaee1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -9,14 +9,12 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Responsible for providing data from an application subtree in zookeeper. * (i.e. /config/v2/tenants/x/session/<session id for an application>/). * - * Takes care of - * - * * @author Tony Vaagenes */ public class ZKApplication { @@ -84,6 +82,10 @@ public class ZKApplication { return reader(data); } + Optional getOptionalDataReader(String path, String node) { + return Optional.ofNullable(getData(path, node)).map(data -> reader(data)); + } + public String getData(String path, String node) { try { return zk.getData(getFullPath(path), node); @@ -181,10 +183,9 @@ public class ZKApplication { } Reader getDataReader(String path) { - final String data = getData(path); - if (data == null) { + String data = getData(path); + if (data == null) throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); - } return reader(data); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index bcb958c4b58..c7ec2657996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public List searchDefinitionContents() { - List ret = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) { - ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd)))); - } + List schemas = new ArrayList<>(); + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); } - return ret; + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); + } + return schemas; } @Override @@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage { try { return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e); + throw new IllegalArgumentException("Could not retrieve config definition " + def, e); } } @@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name); + Optional reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); + if (reader.isPresent()) + return reader.get(); + return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 14fa0cb2dbe..91d59bbeb4e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -170,16 +170,16 @@ public class ZooKeeperClientTest { @Test public void search_definitions_written_to_ZK() { - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("music.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("base.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("video.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("book.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("pc.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("laptop.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("product.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("sock.sd").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("foo.expression").getAbsolute())); - assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("bar.expression").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("music.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("base.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("video.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("book.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("pc.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("laptop.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("product.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("sock.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("foo.expression").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("bar.expression").getAbsolute())); } private Path appPath() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 97d90d8b6b6..d730b21762d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -60,7 +60,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); - assertEquals(deployState.getSearchDefinitions().size(), 5); + assertEquals(deployState.getSchemas().size(), 5); assertEquals(zkApp.searchDefinitionContents().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3); -- cgit v1.2.3