summaryrefslogtreecommitdiffstats
path: root/configserver
diff options
context:
space:
mode:
authorHarald Musum <musum@yahooinc.com>2023-08-18 14:20:22 +0200
committerHarald Musum <musum@yahooinc.com>2023-08-18 14:20:22 +0200
commitc2a6494f6b16bd053f28ecae0f9dedc9e58b6319 (patch)
treecd82d51ba3d8efb9bf874c2f48f6d73c7030055d /configserver
parent5c51af58b98c987975e3db62d9ed09d72b8188d8 (diff)
Make ZooKeeperClient inner class of the only class that uses it
Diffstat (limited to 'configserver')
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java281
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java302
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java10
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java174
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java159
6 files changed, 449 insertions, 479 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
deleted file mode 100644
index efa62625159..00000000000
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ /dev/null
@@ -1,281 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.deploy;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.application.api.ApplicationFile;
-import com.yahoo.config.application.api.ApplicationMetaData;
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.application.api.FileRegistry;
-import com.yahoo.config.application.api.UnparsedConfigDefinition;
-import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.config.provision.serialization.AllocatedHostsSerializer;
-import com.yahoo.io.reader.NamedReader;
-import com.yahoo.path.Path;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.config.ConfigDefinitionKey;
-import com.yahoo.vespa.config.server.filedistribution.FileDBRegistry;
-import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.yolean.Exceptions;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-
-import static com.yahoo.config.application.api.ApplicationPackage.*;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
-
-/**
- * Reads and writes application package to and from ZooKeeper.
- *
- * @author hmusum
- */
-public class ZooKeeperClient {
-
- private final Curator curator;
- private final DeployLogger logger;
- private final Path sessionPath; // session id
-
- private static final ApplicationFile.PathFilter xmlFilter = path -> path.getName().endsWith(".xml");
-
- public ZooKeeperClient(Curator curator, DeployLogger logger, Path sessionPath) {
- this.curator = curator;
- this.logger = logger;
- this.sessionPath = sessionPath;
- }
-
- /**
- * Sets up basic node structure in ZooKeeper and purges old data.
- * This is the first operation on ZK during deploy.
- */
- void initialize() {
- curator.create(sessionPath);
-
- for (String subPath : Arrays.asList(DEFCONFIGS_ZK_SUBPATH,
- USER_DEFCONFIGS_ZK_SUBPATH,
- USERAPP_ZK_SUBPATH,
- ZKApplicationPackage.fileRegistryNode)) {
- // TODO: The replaceFirst below is hackish.
- curator.create(getZooKeeperAppPath().append(subPath.replaceFirst("/", "")));
- }
- }
-
- /**
- * Writes def files and user config into ZK.
- *
- * @param app the application package to feed to zookeeper
- */
- void writeApplicationPackage(ApplicationPackage app) {
- try {
- writeUserDefs(app);
- writeSomeOf(app);
- writeSchemas(app);
- writeUserIncludeDirs(app, app.getUserIncludeDirs());
- writeMetadata(app.getMetaData());
- } catch (Exception e) {
- throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
- "Please ensure that config server is started " +
- "and check the vespa log for configserver errors. ", e);
- }
- }
-
- private void writeSchemas(ApplicationPackage app) throws IOException {
- Collection<NamedReader> schemas = app.getSchemas();
- if (schemas.isEmpty()) return;
-
- Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR);
- curator.create(zkPath);
- // Ensures that ranking expressions and other files are also written
- writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath);
- writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath);
- for (NamedReader sd : schemas) {
- curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader())));
- sd.getReader().close();
- }
- }
-
- /**
- * Writes some application package files into ZK - see write(app).
- *
- * @param app the application package to use as input.
- * @throws java.io.IOException if not able to write to Zookeeper
- */
- private void writeSomeOf(ApplicationPackage app) throws IOException {
- // TODO: We should have a way of doing this which doesn't require repeating all the content
- writeFile(app.getFile(Path.fromString(SERVICES)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
- writeFile(app.getFile(Path.fromString(HOSTS)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
- writeFile(app.getFile(Path.fromString(DEPLOYMENT_FILE.getName())), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
- writeFile(app.getFile(Path.fromString(VALIDATION_OVERRIDES.getName())), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
- writeDir(app.getFile(RULES_DIR),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(RULES_DIR),
- (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX));
- writeDir(app.getFile(QUERY_PROFILES_DIR),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(QUERY_PROFILES_DIR),
- xmlFilter);
- writeDir(app.getFile(PAGE_TEMPLATES_DIR),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(PAGE_TEMPLATES_DIR),
- xmlFilter);
- writeDir(app.getFile(Path.fromString(SEARCHCHAINS_DIR)),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SEARCHCHAINS_DIR),
- xmlFilter);
- writeDir(app.getFile(Path.fromString(DOCPROCCHAINS_DIR)),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(DOCPROCCHAINS_DIR),
- xmlFilter);
- writeDir(app.getFile(Path.fromString(ROUTINGTABLES_DIR)),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ROUTINGTABLES_DIR),
- xmlFilter);
- writeDir(app.getFile(MODELS_GENERATED_REPLICATED_DIR),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(MODELS_GENERATED_REPLICATED_DIR));
- writeDir(app.getFile(SECURITY_DIR),
- getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SECURITY_DIR));
- }
-
- private void writeDir(ApplicationFile file, Path zooKeeperAppPath) throws IOException {
- writeDir(file, zooKeeperAppPath, (__) -> true);
- }
-
- private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter) throws IOException {
- if ( ! dir.isDirectory()) return;
- for (ApplicationFile file : listFiles(dir, filenameFilter)) {
- String name = file.getPath().getName();
- if (name.startsWith(".")) continue; //.svn , .git ...
- if (file.isDirectory()) {
- curator.create(path.append(name));
- writeDir(file, path.append(name), filenameFilter);
- } else {
- writeFile(file, path);
- }
- }
- }
-
- /**
- * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)}
- * with slightly different semantics: Never filter out directories.
- */
- private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) {
- List<ApplicationFile> rawList = dir.listFiles();
- List<ApplicationFile> ret = new ArrayList<>();
- if (rawList != null) {
- for (ApplicationFile f : rawList) {
- if (f.isDirectory()) {
- ret.add(f);
- } else {
- if (filter.accept(f.getPath())) {
- ret.add(f);
- }
- }
- }
- }
- return ret;
- }
-
- private void writeFile(ApplicationFile file, Path zkPath) throws IOException {
- if ( ! file.exists()) return;
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (InputStream inputStream = file.createInputStream()) {
- inputStream.transferTo(baos);
- baos.flush();
- curator.set(zkPath.append(file.getPath().getName()), baos.toByteArray());
- }
- }
-
- private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException {
- for (String userInclude : userIncludeDirs) {
- ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude));
- final List<ApplicationFile> files = dir.listFiles();
- if (files == null || files.isEmpty()) {
- curator.create(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude));
- }
- writeDir(dir, getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude), xmlFilter);
- }
- }
-
- /**
- * Feeds all user-defined .def file from the application package into ZooKeeper (both into
- * /defconfigs and /userdefconfigs
- */
- private void writeUserDefs(ApplicationPackage applicationPackage) {
- Map<ConfigDefinitionKey, UnparsedConfigDefinition> configDefs = applicationPackage.getAllExistingConfigDefs();
- for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) {
- ConfigDefinitionKey key = entry.getKey();
- String contents = entry.getValue().getUnparsedContent();
- writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH), contents);
- writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH), contents);
- }
- logger.log(Level.FINE, configDefs.size() + " user config definitions");
- }
-
- private void writeConfigDefinition(String name, String namespace, Path path, String data) {
- curator.set(path.append(namespace + "." + name), Utf8.toBytes(data));
- }
-
- private void write(Version vespaVersion, FileRegistry fileRegistry) {
- String exportedRegistry = FileDBRegistry.exportRegistry(fileRegistry);
- curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()),
- Utf8.toBytes(exportedRegistry));
- }
-
- /**
- * Feeds application metadata to zookeeper. Used by config model to create config
- * for application metadata
- *
- * @param metaData The application metadata.
- */
- private void writeMetadata(ApplicationMetaData metaData) {
- curator.set(getZooKeeperAppPath(META_ZK_PATH), metaData.asJsonBytes());
- }
-
- void cleanupZooKeeper() {
- try {
- List.of(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH)
- .forEach(path -> curator.delete(getZooKeeperAppPath(path)));
- } catch (Exception e) {
- logger.log(Level.WARNING, "Could not clean up in zookeeper: " + Exceptions.toMessageString(e));
- // Might be called in an exception handler before re-throw, so do not throw here.
- }
- }
-
- /**
- * Gets a full ZK application path
- *
- * @return a String with the full ZK application path
- */
- private Path getZooKeeperAppPath() {
- return getZooKeeperAppPath(null);
- }
-
- /**
- * Gets a full ZK application path
- *
- * @param trailingPath trailing part of path to be appended to ZK app path
- * @return a String with the full ZK application path including trailing path, if set
- */
- private Path getZooKeeperAppPath(String trailingPath) {
- if (trailingPath == null) return sessionPath;
-
- return sessionPath.append(trailingPath);
- }
-
- public void write(AllocatedHosts hosts) throws IOException {
- curator.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode),
- AllocatedHostsSerializer.toJson(hosts));
- }
-
- public void write(Map<Version, FileRegistry> fileRegistryMap) {
- for (Map.Entry<Version, FileRegistry> versionFileRegistryEntry : fileRegistryMap.entrySet()) {
- write(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue());
- }
- }
-
-}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
index 8d889b1fb2c..cb50bd54d38 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
@@ -1,13 +1,53 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.deploy;
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.application.api.ApplicationMetaData;
import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.application.api.UnparsedConfigDefinition;
import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.serialization.AllocatedHostsSerializer;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.vespa.config.server.filedistribution.FileDBRegistry;
+import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.yolean.Exceptions;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
import java.util.Map;
+import java.util.logging.Level;
+
+import static com.yahoo.config.application.api.ApplicationPackage.DEPLOYMENT_FILE;
+import static com.yahoo.config.application.api.ApplicationPackage.DOCPROCCHAINS_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.HOSTS;
+import static com.yahoo.config.application.api.ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.PAGE_TEMPLATES_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.QUERY_PROFILES_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.ROUTINGTABLES_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.RULES_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.SCHEMAS_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.SEARCHCHAINS_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.SECURITY_DIR;
+import static com.yahoo.config.application.api.ApplicationPackage.SERVICES;
+import static com.yahoo.config.application.api.ApplicationPackage.VALIDATION_OVERRIDES;
+import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.getSessionPath;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
/**
* Interface for initializing zookeeper and deploying an application package to zookeeper.
@@ -16,10 +56,11 @@ import java.util.Map;
*/
public class ZooKeeperDeployer {
- private final ZooKeeperClient zooKeeperClient;
+ private final Client client;
- public ZooKeeperDeployer(ZooKeeperClient client) {
- this.zooKeeperClient = client;
+ public ZooKeeperDeployer(Curator curator, DeployLogger logger, ApplicationId applicationId, long sessionId) {
+ Path sessionPath = getSessionPath(applicationId.tenant(), sessionId);
+ this.client = new Client(curator, logger, sessionPath);
}
/**
@@ -32,14 +73,259 @@ public class ZooKeeperDeployer {
*/
public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap,
AllocatedHosts allocatedHosts) throws IOException {
- zooKeeperClient.initialize();
- zooKeeperClient.writeApplicationPackage(applicationPackage);
- zooKeeperClient.write(fileRegistryMap);
- zooKeeperClient.write(allocatedHosts);
+ client.initialize();
+ client.writeApplicationPackage(applicationPackage);
+ client.write(fileRegistryMap);
+ client.write(allocatedHosts);
}
public void cleanup() {
- zooKeeperClient.cleanupZooKeeper();
+ client.cleanupZooKeeper();
+ }
+
+ /**
+ * Reads and writes application package to and from ZooKeeper.
+ *
+ * @author hmusum
+ */
+ public static class Client {
+
+ private final Curator curator;
+ private final DeployLogger logger;
+ private final Path sessionPath; // session id
+
+ private static final ApplicationFile.PathFilter xmlFilter = path -> path.getName().endsWith(".xml");
+
+ public Client(Curator curator, DeployLogger logger, Path sessionPath) {
+ this.curator = curator;
+ this.logger = logger;
+ this.sessionPath = sessionPath;
+ }
+
+ /**
+ * Sets up basic node structure in ZooKeeper and purges old data.
+ * This is the first operation on ZK during deploy.
+ */
+ void initialize() {
+ curator.create(sessionPath);
+
+ for (String subPath : Arrays.asList(DEFCONFIGS_ZK_SUBPATH,
+ USER_DEFCONFIGS_ZK_SUBPATH,
+ USERAPP_ZK_SUBPATH,
+ ZKApplicationPackage.fileRegistryNode)) {
+ // TODO: The replaceFirst below is hackish.
+ curator.create(getZooKeeperAppPath().append(subPath.replaceFirst("/", "")));
+ }
+ }
+
+ /**
+ * Writes def files and user config into ZK.
+ *
+ * @param app the application package to feed to zookeeper
+ */
+ void writeApplicationPackage(ApplicationPackage app) {
+ try {
+ writeUserDefs(app);
+ writeSomeOf(app);
+ writeSchemas(app);
+ writeUserIncludeDirs(app, app.getUserIncludeDirs());
+ writeMetadata(app.getMetaData());
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
+ "Please ensure that config server is started " +
+ "and check the vespa log for configserver errors. ", e);
+ }
+ }
+
+ private void writeSchemas(ApplicationPackage app) throws IOException {
+ Collection<NamedReader> schemas = app.getSchemas();
+ if (schemas.isEmpty()) return;
+
+ Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR);
+ curator.create(zkPath);
+ // Ensures that ranking expressions and other files are also written
+ writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath);
+ writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath);
+ for (NamedReader sd : schemas) {
+ curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader())));
+ sd.getReader().close();
+ }
+ }
+
+ /**
+ * Writes some application package files into ZK - see write(app).
+ *
+ * @param app the application package to use as input.
+ * @throws IOException if not able to write to Zookeeper
+ */
+ private void writeSomeOf(ApplicationPackage app) throws IOException {
+ // TODO: We should have a way of doing this which doesn't require repeating all the content
+ writeFile(app.getFile(Path.fromString(SERVICES)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
+ writeFile(app.getFile(Path.fromString(HOSTS)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
+ writeFile(app.getFile(Path.fromString(DEPLOYMENT_FILE.getName())), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
+ writeFile(app.getFile(Path.fromString(VALIDATION_OVERRIDES.getName())), getZooKeeperAppPath(USERAPP_ZK_SUBPATH));
+ writeDir(app.getFile(RULES_DIR),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(RULES_DIR),
+ (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX));
+ writeDir(app.getFile(QUERY_PROFILES_DIR),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(QUERY_PROFILES_DIR),
+ xmlFilter);
+ writeDir(app.getFile(PAGE_TEMPLATES_DIR),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(PAGE_TEMPLATES_DIR),
+ xmlFilter);
+ writeDir(app.getFile(Path.fromString(SEARCHCHAINS_DIR)),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SEARCHCHAINS_DIR),
+ xmlFilter);
+ writeDir(app.getFile(Path.fromString(DOCPROCCHAINS_DIR)),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(DOCPROCCHAINS_DIR),
+ xmlFilter);
+ writeDir(app.getFile(Path.fromString(ROUTINGTABLES_DIR)),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ROUTINGTABLES_DIR),
+ xmlFilter);
+ writeDir(app.getFile(MODELS_GENERATED_REPLICATED_DIR),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(MODELS_GENERATED_REPLICATED_DIR));
+ writeDir(app.getFile(SECURITY_DIR),
+ getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SECURITY_DIR));
+ }
+
+ private void writeDir(ApplicationFile file, Path zooKeeperAppPath) throws IOException {
+ writeDir(file, zooKeeperAppPath, (__) -> true);
+ }
+
+ private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter) throws IOException {
+ if ( ! dir.isDirectory()) return;
+ for (ApplicationFile file : listFiles(dir, filenameFilter)) {
+ String name = file.getPath().getName();
+ if (name.startsWith(".")) continue; //.svn , .git ...
+ if (file.isDirectory()) {
+ curator.create(path.append(name));
+ writeDir(file, path.append(name), filenameFilter);
+ } else {
+ writeFile(file, path);
+ }
+ }
+ }
+
+ /**
+ * Like {@link ApplicationFile#listFiles(ApplicationFile.PathFilter)}
+ * with slightly different semantics: Never filter out directories.
+ */
+ private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) {
+ List<ApplicationFile> rawList = dir.listFiles();
+ List<ApplicationFile> ret = new ArrayList<>();
+ if (rawList != null) {
+ for (ApplicationFile f : rawList) {
+ if (f.isDirectory()) {
+ ret.add(f);
+ } else {
+ if (filter.accept(f.getPath())) {
+ ret.add(f);
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ private void writeFile(ApplicationFile file, Path zkPath) throws IOException {
+ if ( ! file.exists()) return;
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (InputStream inputStream = file.createInputStream()) {
+ inputStream.transferTo(baos);
+ baos.flush();
+ curator.set(zkPath.append(file.getPath().getName()), baos.toByteArray());
+ }
+ }
+
+ private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException {
+ for (String userInclude : userIncludeDirs) {
+ ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude));
+ final List<ApplicationFile> files = dir.listFiles();
+ if (files == null || files.isEmpty()) {
+ curator.create(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude));
+ }
+ writeDir(dir, getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude), xmlFilter);
+ }
+ }
+
+ /**
+ * Feeds all user-defined .def file from the application package into ZooKeeper (both into
+ * /defconfigs and /userdefconfigs
+ */
+ private void writeUserDefs(ApplicationPackage applicationPackage) {
+ Map<ConfigDefinitionKey, UnparsedConfigDefinition> configDefs = applicationPackage.getAllExistingConfigDefs();
+ for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) {
+ ConfigDefinitionKey key = entry.getKey();
+ String contents = entry.getValue().getUnparsedContent();
+ writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH), contents);
+ writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH), contents);
+ }
+ logger.log(Level.FINE, configDefs.size() + " user config definitions");
+ }
+
+ private void writeConfigDefinition(String name, String namespace, Path path, String data) {
+ curator.set(path.append(namespace + "." + name), Utf8.toBytes(data));
+ }
+
+ private void write(Version vespaVersion, FileRegistry fileRegistry) {
+ String exportedRegistry = FileDBRegistry.exportRegistry(fileRegistry);
+ curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()),
+ Utf8.toBytes(exportedRegistry));
+ }
+
+ /**
+ * Feeds application metadata to zookeeper. Used by config model to create config
+ * for application metadata
+ *
+ * @param metaData The application metadata.
+ */
+ private void writeMetadata(ApplicationMetaData metaData) {
+ curator.set(getZooKeeperAppPath(META_ZK_PATH), metaData.asJsonBytes());
+ }
+
+ void cleanupZooKeeper() {
+ try {
+ List.of(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH)
+ .forEach(path -> curator.delete(getZooKeeperAppPath(path)));
+ } catch (Exception e) {
+ logger.log(Level.WARNING, "Could not clean up in zookeeper: " + Exceptions.toMessageString(e));
+ // Might be called in an exception handler before re-throw, so do not throw here.
+ }
+ }
+
+ /**
+ * Gets a full ZK application path
+ *
+ * @return a String with the full ZK application path
+ */
+ private Path getZooKeeperAppPath() {
+ return getZooKeeperAppPath(null);
+ }
+
+ /**
+ * Gets a full ZK application path
+ *
+ * @param trailingPath trailing part of path to be appended to ZK app path
+ * @return a String with the full ZK application path including trailing path, if set
+ */
+ private Path getZooKeeperAppPath(String trailingPath) {
+ if (trailingPath == null) return sessionPath;
+
+ return sessionPath.append(trailingPath);
+ }
+
+ public void write(AllocatedHosts hosts) throws IOException {
+ curator.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode),
+ AllocatedHostsSerializer.toJson(hosts));
+ }
+
+ public void write(Map<Version, FileRegistry> fileRegistryMap) {
+ for (Map.Entry<Version, FileRegistry> versionFileRegistryEntry : fileRegistryMap.entrySet()) {
+ write(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue());
+ }
+ }
+
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 8d45ac7e8f1..60b8b4d1ea8 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -36,7 +36,6 @@ import com.yahoo.vespa.config.server.ConfigServerSpec;
import com.yahoo.vespa.config.server.TimeoutBudget;
import com.yahoo.vespa.config.server.application.ApplicationSet;
import com.yahoo.vespa.config.server.configchange.ConfigChangeActions;
-import com.yahoo.vespa.config.server.deploy.ZooKeeperClient;
import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer;
import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory;
import com.yahoo.vespa.config.server.host.HostValidator;
@@ -74,8 +73,6 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.zip.ZipException;
-import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.getSessionPath;
-
/**
* A SessionPreparer is responsible for preparing a session given an application package.
*
@@ -396,10 +393,9 @@ public class SessionPreparer {
List<X509Certificate> operatorCertificates,
Optional<CloudAccount> cloudAccount,
List<DataplaneToken> dataplaneTokens) {
- Path sessionPath = getSessionPath(applicationId.tenant(), zooKeeperClient.sessionId());
- ZooKeeperDeployer zkDeployer = new ZooKeeperDeployer(new ZooKeeperClient(curator, deployLogger, sessionPath));
+ var zooKeeperDeplyer = new ZooKeeperDeployer(curator, deployLogger, applicationId, zooKeeperClient.sessionId());
try {
- zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts);
+ zooKeeperDeplyer.deploy(applicationPackage, fileRegistryMap, allocatedHosts);
new SessionSerializer().write(zooKeeperClient,
applicationId,
fileReference,
@@ -413,7 +409,7 @@ public class SessionPreparer {
dataplaneTokens,
writeSessionData);
} catch (RuntimeException | IOException e) {
- zkDeployer.cleanup();
+ zooKeeperDeplyer.cleanup();
throw new RuntimeException("Error preparing session", e);
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 7d1a7ceae4e..04856f0b5c4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -361,7 +361,7 @@ public class SessionZooKeeperClient {
transaction.commit();
}
- static Path getSessionPath(TenantName tenantName, long sessionId) {
+ public static Path getSessionPath(TenantName tenantName, long sessionId) {
return TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId));
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
deleted file mode 100644
index 2d42cb12076..00000000000
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.deploy;
-
-import com.google.common.collect.ImmutableSet;
-import com.yahoo.component.Version;
-import com.yahoo.config.application.api.ApplicationMetaData;
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.application.api.FileRegistry;
-import com.yahoo.config.model.application.provider.BaseDeployLogger;
-import com.yahoo.config.model.application.provider.DeployData;
-import com.yahoo.config.model.application.provider.FilesApplicationPackage;
-import com.yahoo.config.model.application.provider.MockFileRegistry;
-import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostSpec;
-import com.yahoo.path.Path;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.mock.MockCurator;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Unit tests for ZooKeeperClient.
- *
- * @author hmusum
- */
-public class ZooKeeperClientTest {
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
-
- private Curator zk;
- private final Path appPath = Path.fromString("/1");
-
- @Before
- public void setupZK() throws IOException {
- zk = new MockCurator();
- ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), appPath);
- ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"),
- new DeployData("/bar/baz",
- ApplicationId.from("default", "appName", "default"),
- 1345L,
- true,
- 3L,
- 2L));
- Map<Version, FileRegistry> fileRegistries = createFileRegistries();
- app.writeMetaData();
- zkc.initialize();
- zkc.writeApplicationPackage(app);
- zkc.write(fileRegistries);
- }
-
- private Map<Version, FileRegistry> createFileRegistries() {
- FileRegistry a = new MockFileRegistry();
- a.addFile("fileA");
- FileRegistry b = new MockFileRegistry();
- b.addFile("fileB");
- Map<Version, FileRegistry> registryMap = new HashMap<>();
- registryMap.put(new Version(1, 2, 3), a);
- registryMap.put(new Version(3, 2, 1), b);
- return registryMap;
- }
-
- @Test
- public void testInitZooKeeper() {
- Curator zk = new MockCurator();
- BaseDeployLogger logger = new BaseDeployLogger();
- long generation = 1L;
- ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, Path.fromString("/1"));
- zooKeeperClient.initialize();
- Path appPath = Path.fromString("/");
- assertEquals(1, zk.getChildren(appPath).size());
- Path currentAppPath = appPath.append(String.valueOf(generation));
- assertTrue(zk.exists(currentAppPath));
- assertTrue(zk.exists(currentAppPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
- assertEquals(4, zk.getChildren(currentAppPath).size());
- }
-
- @Test
- public void testFeedDefFilesToZooKeeper() {
- Path defsPath = appPath.append(DEFCONFIGS_ZK_SUBPATH);
- assertTrue(zk.exists(appPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
- List<String> children = zk.getChildren(defsPath);
- assertEquals(defsPath + " children", 1, children.size());
- Collections.sort(children);
- assertEquals("a.b.test2", children.get(0));
-
- assertTrue(zk.exists(appPath.append(USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
- Path userDefsPath = appPath.append(USER_DEFCONFIGS_ZK_SUBPATH);
- children = zk.getChildren(userDefsPath);
- assertEquals(1, children.size());
- Collections.sort(children);
- assertEquals("a.b.test2", children.get(0));
- }
-
- @Test
- public void testFeedAppMetaDataToZooKeeper() {
- assertTrue(zk.exists(appPath.append(META_ZK_PATH)));
- ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(
- Utf8.toString(zk.getData(appPath.append(META_ZK_PATH)).get()));
- assertTrue(metaData.getChecksum().length() > 0);
- assertTrue(metaData.isInternalRedeploy());
- assertEquals("/bar/baz", metaData.getDeployPath());
- assertEquals(1345, metaData.getDeployTimestamp().longValue());
- assertEquals(3, metaData.getGeneration().longValue());
- assertEquals(2, metaData.getPreviousActiveGeneration());
- }
-
- @Test
- public void testVersionedFileRegistry() {
- Path fileRegPath = appPath.append(ZKApplicationPackage.fileRegistryNode);
- assertTrue(zk.exists(fileRegPath));
- assertTrue(zk.exists(fileRegPath.append("/1.2.3")));
- assertTrue(zk.exists(fileRegPath.append("/3.2.1")));
- // assertNull("Data at " + fileRegPath, zk.getData(fileRegPath)); Not null any more .. hm
- }
-
- @Test
- public void include_dirs_are_written_to_ZK() {
- assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("dir1").append("default.xml")));
- assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain2.xml")));
- assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain3.xml")));
- }
-
- @Test
- public void search_chain_dir_written_to_ZK() {
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml")));
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml")));
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml")));
- }
-
- private Path appPath() {
- return appPath.append(USERAPP_ZK_SUBPATH);
- }
-
- @Test
- public void testWritingHostNamesToZooKeeper() throws IOException {
- Curator zk = new MockCurator();
- BaseDeployLogger logger = new BaseDeployLogger();
- Path app = Path.fromString("/1");
- ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app);
- zooKeeperClient.initialize();
- HostSpec host1 = new HostSpec("host1.yahoo.com", Optional.empty());
- HostSpec host2 = new HostSpec("host2.yahoo.com", Optional.empty());
- ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2);
- zooKeeperClient.write(AllocatedHosts.withHosts(hosts));
- Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode);
- assertTrue(zk.exists(hostsPath));
-
- AllocatedHosts deserialized = fromJson(zk.getData(hostsPath).get());
- assertEquals(hosts, deserialized.getHosts());
- }
-
-}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
index 1bc980c9099..17344e94c51 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
@@ -1,16 +1,26 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.deploy;
+import com.google.common.collect.ImmutableSet;
import com.yahoo.component.Version;
+import com.yahoo.config.application.api.ApplicationMetaData;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.application.provider.DeployData;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostSpec;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -18,8 +28,20 @@ import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.logging.Level;
+import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson;
+import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.getSessionPath;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -28,10 +50,31 @@ import static org.junit.Assert.fail;
*/
public class ZooKeeperDeployerTest {
+ private Curator zk;
+ private final Path appPath = Path.fromString("/1");
+
@Rule
public TemporaryFolder folder = new TemporaryFolder();
private static final String defFile = "test2.def";
+ @Before
+ public void setupZK() throws IOException {
+ zk = new MockCurator();
+ ZooKeeperDeployer.Client zkc = new ZooKeeperDeployer.Client(zk, new BaseDeployLogger(), appPath);
+ ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"),
+ new DeployData("/bar/baz",
+ ApplicationId.from("default", "appName", "default"),
+ 1345L,
+ true,
+ 3L,
+ 2L));
+ Map<Version, FileRegistry> fileRegistries = createFileRegistries();
+ app.writeMetaData();
+ zkc.initialize();
+ zkc.writeApplicationPackage(app);
+ zkc.write(fileRegistries);
+ }
+
@Test
public void require_that_deployer_is_initialized() throws IOException {
Curator curator = new MockCurator();
@@ -43,17 +86,117 @@ public class ZooKeeperDeployerTest {
e.printStackTrace();
fail();
}
- deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/1"));
- deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/2"));
+ deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, 1);
+ deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, 2);
+ }
+
+ private Map<Version, FileRegistry> createFileRegistries() {
+ FileRegistry a = new MockFileRegistry();
+ a.addFile("fileA");
+ FileRegistry b = new MockFileRegistry();
+ b.addFile("fileB");
+ Map<Version, FileRegistry> registryMap = new HashMap<>();
+ registryMap.put(new Version(1, 2, 3), a);
+ registryMap.put(new Version(3, 2, 1), b);
+ return registryMap;
+ }
+
+ @Test
+ public void testInitZooKeeper() {
+ Curator zk = new MockCurator();
+ BaseDeployLogger logger = new BaseDeployLogger();
+ long generation = 1L;
+ ZooKeeperDeployer.Client client = new ZooKeeperDeployer.Client(zk, logger, Path.fromString("/1"));
+ client.initialize();
+ Path appPath = Path.fromString("/");
+ assertEquals(1, zk.getChildren(appPath).size());
+ Path currentAppPath = appPath.append(String.valueOf(generation));
+ assertTrue(zk.exists(currentAppPath));
+ assertTrue(zk.exists(currentAppPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
+ assertEquals(4, zk.getChildren(currentAppPath).size());
+ }
+
+ @Test
+ public void testFeedDefFilesToZooKeeper() {
+ Path defsPath = appPath.append(DEFCONFIGS_ZK_SUBPATH);
+ assertTrue(zk.exists(appPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
+ List<String> children = zk.getChildren(defsPath);
+ assertEquals(defsPath + " children", 1, children.size());
+ Collections.sort(children);
+ assertEquals("a.b.test2", children.get(0));
+
+ assertTrue(zk.exists(appPath.append(USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
+ Path userDefsPath = appPath.append(USER_DEFCONFIGS_ZK_SUBPATH);
+ children = zk.getChildren(userDefsPath);
+ assertEquals(1, children.size());
+ Collections.sort(children);
+ assertEquals("a.b.test2", children.get(0));
+ }
+
+ @Test
+ public void testFeedAppMetaDataToZooKeeper() {
+ assertTrue(zk.exists(appPath.append(META_ZK_PATH)));
+ ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(
+ Utf8.toString(zk.getData(appPath.append(META_ZK_PATH)).get()));
+ assertTrue(metaData.getChecksum().length() > 0);
+ assertTrue(metaData.isInternalRedeploy());
+ assertEquals("/bar/baz", metaData.getDeployPath());
+ assertEquals(1345, metaData.getDeployTimestamp().longValue());
+ assertEquals(3, metaData.getGeneration().longValue());
+ assertEquals(2, metaData.getPreviousActiveGeneration());
+ }
+
+ @Test
+ public void testVersionedFileRegistry() {
+ Path fileRegPath = appPath.append(ZKApplicationPackage.fileRegistryNode);
+ assertTrue(zk.exists(fileRegPath));
+ assertTrue(zk.exists(fileRegPath.append("/1.2.3")));
+ assertTrue(zk.exists(fileRegPath.append("/3.2.1")));
+ // assertNull("Data at " + fileRegPath, zk.getData(fileRegPath)); Not null any more .. hm
+ }
+
+ @Test
+ public void include_dirs_are_written_to_ZK() {
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("dir1").append("default.xml")));
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain2.xml")));
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain3.xml")));
+ }
+
+ @Test
+ public void search_chain_dir_written_to_ZK() {
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml")));
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml")));
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml")));
+ }
+
+ private Path appPath() {
+ return appPath.append(USERAPP_ZK_SUBPATH);
+ }
+
+ @Test
+ public void testWritingHostNamesToZooKeeper() throws IOException {
+ Curator zk = new MockCurator();
+ BaseDeployLogger logger = new BaseDeployLogger();
+ Path app = Path.fromString("/1");
+ ZooKeeperDeployer.Client client = new ZooKeeperDeployer.Client(zk, logger, app);
+ client.initialize();
+ HostSpec host1 = new HostSpec("host1.yahoo.com", Optional.empty());
+ HostSpec host2 = new HostSpec("host2.yahoo.com", Optional.empty());
+ ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2);
+ client.write(AllocatedHosts.withHosts(hosts));
+ Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode);
+ assertTrue(zk.exists(hostsPath));
+
+ AllocatedHosts deserialized = fromJson(zk.getData(hostsPath).get());
+ assertEquals(hosts, deserialized.getHosts());
}
- public void deploy(ApplicationPackage applicationPackage, Curator curator, Path appPath) throws IOException {
- MockDeployLogger logger = new MockDeployLogger();
- ZooKeeperClient client = new ZooKeeperClient(curator, logger, appPath);
- ZooKeeperDeployer deployer = new ZooKeeperDeployer(client);
+ public void deploy(ApplicationPackage applicationPackage, Curator curator, long sessionId) throws IOException {
+ ZooKeeperDeployer deployer = new ZooKeeperDeployer(curator, new MockDeployLogger(), applicationPackage.getApplicationId(), sessionId);
+ deployer.deploy(applicationPackage, Map.of(new Version(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Set.of()));
- deployer.deploy(applicationPackage, Collections.singletonMap(new Version(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet()));
- assertTrue(curator.exists(appPath));
+ Path sessionPath = getSessionPath(applicationPackage.getApplicationId().tenant(), sessionId);
+ assertTrue(curator.exists(sessionPath));
}
private static class MockDeployLogger implements DeployLogger {