Merge pull request #12563 from vespa-engine/bratseth/search-to-schema

Bratseth/search to schema

5 files changed, 64 insertions, 77 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 13ef19f5f5d..9cb97b9cfb5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -87,17 +87,16 @@ public class ZooKeeperClient {
 
     /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */
     private void createZooKeeperNodes() {
-        if (!configCurator.exists(rootPath.getAbsolute())) {
+        if ( ! configCurator.exists(rootPath.getAbsolute()))
             configCurator.createNode(rootPath.getAbsolute());
-        }
 
-        for (String subPath : Arrays.asList(
-                ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
-                ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
-                ConfigCurator.USERAPP_ZK_SUBPATH,
-                ZKApplicationPackage.fileRegistryNode)) {
-            // TODO The replaceFirst below is hackish.
-            configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", ""));
+        for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
+                                            ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
+                                            ConfigCurator.USERAPP_ZK_SUBPATH,
+                                            ZKApplicationPackage.fileRegistryNode)) {
+            // TODO: The replaceFirst below is hackish.
+            configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(),
+                                     subPath.replaceFirst("/", ""));
         }
     }
 
@@ -108,7 +107,6 @@ public class ZooKeeperClient {
      */
     void write(ApplicationPackage app) {
         logFine("Feeding application config into ZooKeeper");
-        // gives lots and lots of debug output: // BasicConfigurator.configure();
         try {
             logFine("Feeding user def files into ZooKeeper");
             writeUserDefs(app);
@@ -121,43 +119,33 @@ public class ZooKeeperClient {
             write(app.getMetaData());
         } catch (Exception e) {
             throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
-                    "Please ensure that cloudconfig_server is started on the config server node(s), " +
-                    "and check the vespa log for configserver errors. ", e);
+                                            "Please ensure that cloudconfig_server is started on the config server node(s), " +
+                                            "and check the vespa log for configserver errors. ", e);
         }
     }
 
     private void writeSearchDefinitions(ApplicationPackage app) throws IOException {
         Collection<NamedReader> sds = app.getSearchDefinitions();
-        if (sds.isEmpty()) {
-            return;
-        }
-        Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
+        if (sds.isEmpty()) return;
+
+        Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SCHEMAS_DIR);
         configCurator.createNode(zkPath.getAbsolute());
-        // Ensures that ranking expressions and other files are also fed.
+        // Ensures that ranking expressions and other files are also written
         writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
+        writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false);
         for (NamedReader sd : sds) {
-            String name = sd.getName();
-            Reader reader = sd.getReader();
-            String data = com.yahoo.io.IOUtils.readAll(reader);
-            reader.close();
-            configCurator.putData(zkPath.getAbsolute(), name, data);
+            configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader()));
+            sd.getReader().close();
         }
     }
 
     /**
      * Puts some of the application package files into ZK - see write(app).
      *
-     * @param app The application package to use as input.
-     * @throws java.io.IOException  if not able to write to Zookeeper
+     * @param app the application package to use as input.
+     * @throws java.io.IOException if not able to write to Zookeeper
      */
     private void writeSomeOf(ApplicationPackage app) throws IOException {
-        ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() {
-            @Override
-            public boolean accept(Path path) {
-                return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX);
-            }
-        };
-        // Copy app package files and subdirs into zk
         // TODO: We should have a way of doing this which doesn't require repeating all the content
         writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)),
                   getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
@@ -169,7 +157,8 @@ public class ZooKeeperClient {
                   getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
         writeDir(app.getFile(ApplicationPackage.RULES_DIR),
                  getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR),
-                 srFilter, true);
+                 (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX),
+                 true);
         writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR),
                  getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR),
                  xmlFilter, true);
@@ -194,20 +183,12 @@ public class ZooKeeperClient {
     }
 
     private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException {
-        writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() {
-            @Override
-            public boolean accept(Path path) {
-                return true;
-            }
-        }, recurse);
+        writeDir(file, zooKeeperAppPath, (__) -> true, recurse);
     }
 
     private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException {
-        if (!dir.isDirectory()) {
-            logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper.");
-            return;
-        }
-        for (ApplicationFile file: listFiles(dir, filenameFilter)) {
+        if ( ! dir.isDirectory()) return;
+        for (ApplicationFile file : listFiles(dir, filenameFilter)) {
             String name = file.getPath().getName();
             if (name.startsWith(".")) continue; //.svn , .git ...
             if ("CVS".equals(name)) continue;
@@ -223,7 +204,8 @@ public class ZooKeeperClient {
     }
 
     /**
-     * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories.
+     * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)}
+     * with slightly different semantics: Never filter out directories.
      */
     private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) {
         List<ApplicationFile> rawList = dir.listFiles();
@@ -243,9 +225,8 @@ public class ZooKeeperClient {
     }
 
     private void writeFile(ApplicationFile file, Path zkPath) throws IOException {
-        if (!file.exists()) {
-            return;
-        }
+        if ( ! file.exists()) return;
+
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         try (InputStream inputStream = file.createInputStream()) {
             inputStream.transferTo(baos);
@@ -292,8 +273,8 @@ public class ZooKeeperClient {
         String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
 
         configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(),
-                vespaVersion.toFullString(),
-                exportedRegistry);
+                              vespaVersion.toFullString(),
+                              exportedRegistry);
     }
 
     /**
@@ -343,9 +324,8 @@ public class ZooKeeperClient {
     }
 
     public void write(AllocatedHosts hosts) throws IOException {
-        configCurator.putData(
-                rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
-                AllocatedHostsSerializer.toJson(hosts));
+        configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
+                              AllocatedHostsSerializer.toJson(hosts));
     }
 
     public void write(Map<Version, FileRegistry> fileRegistryMap) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
index 3ae678969eb..5f180bdaee1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
@@ -9,14 +9,12 @@ import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Responsible for providing data from an application subtree in zookeeper.
  * (i.e. /config/v2/tenants/x/session/&lt;session id for an application&gt;/).
  *
- * Takes care of
- *
- *
  * @author Tony Vaagenes
  */
 public class ZKApplication {
@@ -84,6 +82,10 @@ public class ZKApplication {
         return reader(data);
     }
 
+    Optional<Reader> getOptionalDataReader(String path, String node) {
+        return Optional.ofNullable(getData(path, node)).map(data -> reader(data));
+    }
+
     public String getData(String path, String node) {
         try {
             return zk.getData(getFullPath(path), node);
@@ -181,10 +183,9 @@ public class ZKApplication {
     }
 
     Reader getDataReader(String path) {
-        final String data = getData(path);
-        if (data == null) {
+        String data = getData(path);
+        if (data == null)
             throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists");
-        }
         return reader(data);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index bcb958c4b58..c7ec2657996 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage {
 
     @Override
     public List<NamedReader> searchDefinitionContents() {
-        List<NamedReader> ret = new ArrayList<>();
-        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) {
-            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) {
-                ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd))));
-            }
+        List<NamedReader> schemas = new ArrayList<>();
+        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
+            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
         }
-        return ret;
+        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
+            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
+        }
+        return schemas;
     }
 
     @Override
@@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
         try {
             return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def);
         } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e);
+            throw new IllegalArgumentException("Could not retrieve config definition " + def, e);
         }
     }
 
@@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage {
 
     @Override
     public Reader getRankingExpression(String name) {
-        return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name);
+        Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name);
+        if (reader.isPresent())
+            return reader.get();
+        return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name);
     }
 
     @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
index 14fa0cb2dbe..91d59bbeb4e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
@@ -170,16 +170,16 @@ public class ZooKeeperClientTest {
 
     @Test
     public void search_definitions_written_to_ZK() {
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("music.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("base.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("video.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("book.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("pc.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("laptop.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("product.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("sock.sd").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("foo.expression").getAbsolute()));
-        assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("bar.expression").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("music.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("base.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("video.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("book.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("pc.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("laptop.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("product.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("sock.sd").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("foo.expression").getAbsolute()));
+        assertTrue(zk.exists(appPath().append(ApplicationPackage.SCHEMAS_DIR).append("bar.expression").getAbsolute()));
     }
 
     private Path appPath() {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
index f2c6aac2bda..e6fa1768024 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
@@ -59,7 +59,7 @@ public class ZKApplicationPackageTest {
         assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches());
         assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches());
         DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build();
-        assertEquals(deployState.getSearchDefinitions().size(), 5);
+        assertEquals(deployState.getSchemas().size(), 5);
         assertEquals(zkApp.searchDefinitionContents().size(), 5);
         assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n");
         assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3);