summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-07-12 15:34:27 +0200
committerGitHub <noreply@github.com>2021-07-12 15:34:27 +0200
commitad255a9f8b2a2cc23d3c0079e87af6878f6176b3 (patch)
tree7103a4dfeb73af45bd46eba8a28eeb47f05f5398
parent2f69a54cc50e0604cd3748ee18fb33f8fd525bd0 (diff)
parent65e8e5f0c61bacaad1301dce436f677c5a8f0f91 (diff)
Merge pull request #18584 from vespa-engine/musum/cleanup-ConfigCurator-1
Cleanup Curator and ConfigCurator usage [run-systemtest]
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java19
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java56
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java21
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java139
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java32
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java255
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java30
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java9
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java125
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java39
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java57
-rw-r--r--configserver/src/main/resources/configserver-app/services.xml1
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java17
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java64
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java22
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java55
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java12
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java25
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java7
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java7
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java243
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java12
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java19
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java33
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java2
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java5
-rw-r--r--zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java10
-rw-r--r--zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java9
31 files changed, 406 insertions, 927 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java
index eb57b9346c7..c6ab70adce7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
import com.yahoo.path.Path;
@@ -17,7 +17,7 @@ public class SuperModelGenerationCounter implements GenerationCounter {
private final CuratorCounter counter;
public SuperModelGenerationCounter(Curator curator) {
- this.counter = new CuratorCounter(curator, counterPath.getAbsolute());
+ this.counter = new CuratorCounter(curator, counterPath);
}
/**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java
index 15a9b8f74d0..8097cd24514 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java
@@ -1,16 +1,19 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
import com.google.common.base.Splitter;
import com.yahoo.config.model.api.ConfigDefinitionRepo;
-import java.util.logging.Level;
+import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.buildergen.ConfigDefinition;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.util.ConfigUtils;
+import com.yahoo.vespa.curator.Curator;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Level;
import java.util.logging.Logger;
/**
@@ -24,10 +27,12 @@ public class UserConfigDefinitionRepo implements ConfigDefinitionRepo {
// For testing only
public UserConfigDefinitionRepo() {}
- public UserConfigDefinitionRepo(ConfigCurator configCurator, String appPath) {
- if (configCurator.exists(appPath)) {
- for (String nodeName : configCurator.getChildren(appPath)) {
- String payload = configCurator.getData(appPath, nodeName);
+ public UserConfigDefinitionRepo(Curator curator, Path appPath) {
+ if (curator.exists(appPath)) {
+ for (String nodeName : curator.getChildren(appPath)) {
+ String payload = curator.getData(appPath.append(nodeName))
+ .map(Utf8::toString)
+ .orElseThrow(() -> new IllegalArgumentException("No config definition data at " + nodeName));
ConfigDefinitionKey dKey = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName);
defs.put(dKey, new ConfigDefinition(dKey.getName(), Splitter.on("\n").splitToList(payload).toArray(new String[0])));
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index f0a63757477..99632ec323e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -12,11 +12,11 @@ import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry;
import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.config.provision.serialization.AllocatedHostsSerializer;
import com.yahoo.io.reader.NamedReader;
-import java.util.logging.Level;
import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.yolean.Exceptions;
import java.io.ByteArrayOutputStream;
@@ -27,12 +27,13 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.logging.Level;
import static com.yahoo.config.application.api.ApplicationPackage.*;
-import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.DEFCONFIGS_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.META_ZK_PATH;
-import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USERAPP_ZK_SUBPATH;
-import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
/**
* Reads and writes application package to and from ZooKeeper.
@@ -41,14 +42,14 @@ import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONF
*/
public class ZooKeeperClient {
- private final ConfigCurator configCurator;
+ private final Curator curator;
private final DeployLogger logger;
private final Path sessionPath; // session id
private static final ApplicationFile.PathFilter xmlFilter = path -> path.getName().endsWith(".xml");
- public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, Path sessionPath) {
- this.configCurator = configCurator;
+ public ZooKeeperClient(Curator curator, DeployLogger logger, Path sessionPath) {
+ this.curator = curator;
this.logger = logger;
this.sessionPath = sessionPath;
}
@@ -58,16 +59,14 @@ public class ZooKeeperClient {
* This is the first operation on ZK during deploy.
*/
void initialize() {
- if ( ! configCurator.exists(sessionPath.getAbsolute()))
- configCurator.createNode(sessionPath.getAbsolute());
+ curator.create(sessionPath);
for (String subPath : Arrays.asList(DEFCONFIGS_ZK_SUBPATH,
USER_DEFCONFIGS_ZK_SUBPATH,
USERAPP_ZK_SUBPATH,
ZKApplicationPackage.fileRegistryNode)) {
// TODO: The replaceFirst below is hackish.
- configCurator.createNode(getZooKeeperAppPath().getAbsolute(),
- subPath.replaceFirst("/", ""));
+ curator.create(getZooKeeperAppPath().append(subPath.replaceFirst("/", "")));
}
}
@@ -95,12 +94,12 @@ public class ZooKeeperClient {
if (sds.isEmpty()) return;
Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR);
- configCurator.createNode(zkPath.getAbsolute());
+ curator.create(zkPath);
// Ensures that ranking expressions and other files are also written
writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false);
for (NamedReader sd : sds) {
- configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader()));
+ curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader())));
sd.getReader().close();
}
}
@@ -154,7 +153,7 @@ public class ZooKeeperClient {
String name = file.getPath().getName();
if (name.startsWith(".")) continue; //.svn , .git ...
if (file.isDirectory()) {
- configCurator.createNode(path.append(name).getAbsolute());
+ curator.create(path.append(name));
if (recurse) {
writeDir(file, path.append(name), filenameFilter, recurse);
}
@@ -192,7 +191,7 @@ public class ZooKeeperClient {
try (InputStream inputStream = file.createInputStream()) {
inputStream.transferTo(baos);
baos.flush();
- configCurator.putData(zkPath.append(file.getPath().getName()).getAbsolute(), baos.toByteArray());
+ curator.set(zkPath.append(file.getPath().getName()), baos.toByteArray());
}
}
@@ -201,7 +200,7 @@ public class ZooKeeperClient {
ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude));
final List<ApplicationFile> files = dir.listFiles();
if (files == null || files.isEmpty()) {
- configCurator.createNode(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude).getAbsolute());
+ curator.create(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude));
}
writeDir(dir,
getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude),
@@ -218,21 +217,20 @@ public class ZooKeeperClient {
for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) {
ConfigDefinitionKey key = entry.getKey();
String contents = entry.getValue().getUnparsedContent();
- writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents);
- writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents);
+ writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(USER_DEFCONFIGS_ZK_SUBPATH), contents);
+ writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(DEFCONFIGS_ZK_SUBPATH), contents);
}
logger.log(Level.FINE, configDefs.size() + " user config definitions");
}
- private void writeConfigDefinition(String name, String namespace, String path, String data) {
- configCurator.putDefData(namespace + "." + name, path, com.yahoo.text.Utf8.toBytes(data));
+ private void writeConfigDefinition(String name, String namespace, Path path, String data) {
+ curator.set(path.append(namespace + "." + name), Utf8.toBytes(data));
}
private void write(Version vespaVersion, FileRegistry fileRegistry) {
String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
- configCurator.putData(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).getAbsolute(),
- vespaVersion.toFullString(),
- exportedRegistry);
+ curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()),
+ Utf8.toBytes(exportedRegistry));
}
/**
@@ -242,13 +240,13 @@ public class ZooKeeperClient {
* @param metaData The application metadata.
*/
private void writeMetadata(ApplicationMetaData metaData) {
- configCurator.putData(getZooKeeperAppPath(META_ZK_PATH).getAbsolute(), metaData.asJsonBytes());
+ curator.set(getZooKeeperAppPath(META_ZK_PATH), metaData.asJsonBytes());
}
void cleanupZooKeeper() {
try {
List.of(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH)
- .forEach(path -> configCurator.deleteRecurse(getZooKeeperAppPath(path).getAbsolute()));
+ .forEach(path -> curator.delete(getZooKeeperAppPath(path)));
} catch (Exception e) {
logger.log(Level.WARNING, "Could not clean up in zookeeper: " + Exceptions.toMessageString(e));
//Might be called in an exception handler before re-throw, so do not throw here.
@@ -277,8 +275,8 @@ public class ZooKeeperClient {
}
public void write(AllocatedHosts hosts) throws IOException {
- configCurator.putData(sessionPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
- AllocatedHostsSerializer.toJson(hosts));
+ curator.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode),
+ AllocatedHostsSerializer.toJson(hosts));
}
public void write(Map<Version, FileRegistry> fileRegistryMap) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
index e677d248630..27ee040fe4f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
@@ -37,8 +37,8 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.server.zookeeper.SessionCounter;
+import com.yahoo.vespa.config.server.zookeeper.ZKApplication;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.flags.BooleanFlag;
@@ -114,7 +114,6 @@ public class SessionRepository {
private final SessionPreparer sessionPreparer;
private final Path sessionsPath;
private final TenantName tenantName;
- private final ConfigCurator configCurator;
private final SessionCounter sessionCounter;
private final SecretStore secretStore;
private final HostProvisionerProvider hostProvisionerProvider;
@@ -123,11 +122,12 @@ public class SessionRepository {
private final Zone zone;
private final ModelFactoryRegistry modelFactoryRegistry;
private final ConfigDefinitionRepo configDefinitionRepo;
+ private final int maxNodeSize;
public SessionRepository(TenantName tenantName,
TenantApplications applicationRepo,
SessionPreparer sessionPreparer,
- ConfigCurator configCurator,
+ Curator curator,
Metrics metrics,
StripedExecutor<TenantName> zkWatcherExecutor,
PermanentApplicationPackage permanentApplicationPackage,
@@ -140,13 +140,13 @@ public class SessionRepository {
Zone zone,
Clock clock,
ModelFactoryRegistry modelFactoryRegistry,
- ConfigDefinitionRepo configDefinitionRepo) {
+ ConfigDefinitionRepo configDefinitionRepo,
+ int maxNodeSize) {
this.tenantName = tenantName;
- this.configCurator = configCurator;
- sessionCounter = new SessionCounter(configCurator, tenantName);
+ sessionCounter = new SessionCounter(curator, tenantName);
this.sessionsPath = TenantRepository.getSessionsPath(tenantName);
this.clock = clock;
- this.curator = configCurator.curator();
+ this.curator = curator;
this.sessionLifetime = Duration.ofSeconds(configserverConfig.sessionLifetime());
this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command);
this.permanentApplicationPackage = permanentApplicationPackage;
@@ -163,6 +163,7 @@ public class SessionRepository {
this.zone = zone;
this.modelFactoryRegistry = modelFactoryRegistry;
this.configDefinitionRepo = configDefinitionRepo;
+ this.maxNodeSize = maxNodeSize;
loadSessions(Flags.LOAD_LOCAL_SESSIONS_WHEN_BOOTSTRAPPING.bindTo(flagSource)); // Needs to be done before creating cache below
this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor);
@@ -591,7 +592,7 @@ public class SessionRepository {
private void ensureSessionPathDoesNotExist(long sessionId) {
Path sessionPath = getSessionPath(sessionId);
- if (configCurator.exists(sessionPath.getAbsolute())) {
+ if (curator.exists(sessionPath)) {
throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper");
}
}
@@ -775,12 +776,12 @@ public class SessionRepository {
}
Path getSessionStatePath(long sessionId) {
- return getSessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH);
+ return getSessionPath(sessionId).append(ZKApplication.SESSIONSTATE_ZK_SUBPATH);
}
private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) {
String serverId = configserverConfig.serverId();
- return new SessionZooKeeperClient(curator, configCurator, tenantName, sessionId, serverId);
+ return new SessionZooKeeperClient(curator, tenantName, sessionId, serverId, maxNodeSize);
}
private File getAndValidateExistingSessionAppDir(long sessionId) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 12fcdb6b87d..c7b19968f0a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -24,7 +24,7 @@ import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer;
import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
+import com.yahoo.vespa.config.server.zookeeper.ZKApplication;
import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.transaction.CuratorOperations;
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
import static com.yahoo.vespa.curator.Curator.CompletionWaiter;
import static com.yahoo.yolean.Exceptions.uncheck;
@@ -62,23 +63,23 @@ public class SessionZooKeeperClient {
private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates";
private final Curator curator;
- private final ConfigCurator configCurator;
private final TenantName tenantName;
private final Path sessionPath;
private final Path sessionStatusPath;
private final String serverId; // hostname
+ private final int maxNodeSize;
- public SessionZooKeeperClient(Curator curator,
- ConfigCurator configCurator,
- TenantName tenantName,
- long sessionId,
- String serverId) {
+ public SessionZooKeeperClient(Curator curator, TenantName tenantName, long sessionId, String serverId, int maxNodeSize) {
this.curator = curator;
- this.configCurator = configCurator;
this.tenantName = tenantName;
this.sessionPath = getSessionPath(tenantName, sessionId);
this.serverId = serverId;
- this.sessionStatusPath = sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH);
+ this.sessionStatusPath = sessionPath.append(ZKApplication.SESSIONSTATE_ZK_SUBPATH);
+ this.maxNodeSize = maxNodeSize;
+ }
+
+ public SessionZooKeeperClient(Curator curator, TenantName tenantName, long sessionId, String serverId) {
+ this(curator, tenantName, sessionId, serverId, 10 * 1024 * 1024);
}
public void writeStatus(Session.Status sessionStatus) {
@@ -137,93 +138,93 @@ public class SessionZooKeeperClient {
}
public ApplicationPackage loadApplicationPackage() {
- return new ZKApplicationPackage(configCurator, sessionPath);
+ return new ZKApplicationPackage(curator, sessionPath, maxNodeSize);
}
public ConfigDefinitionRepo getUserConfigDefinitions() {
- return new UserConfigDefinitionRepo(configCurator, sessionPath.append(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute());
+ return new UserConfigDefinitionRepo(curator, sessionPath.append(USER_DEFCONFIGS_ZK_SUBPATH));
}
- private String applicationIdPath() {
- return sessionPath.append(APPLICATION_ID_PATH).getAbsolute();
+ private Path applicationIdPath() {
+ return sessionPath.append(APPLICATION_ID_PATH);
}
public void writeApplicationId(ApplicationId id) {
if ( ! id.tenant().equals(tenantName))
throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenantName + "'");
- configCurator.putData(applicationIdPath(), id.serializedForm());
+ curator.set(applicationIdPath(), Utf8.toBytes(id.serializedForm()));
}
public Optional<ApplicationId> readApplicationId() {
- if ( ! configCurator.exists(applicationIdPath())) return Optional.empty();
- return Optional.of(ApplicationId.fromSerializedForm(configCurator.getData(applicationIdPath())));
+ return curator.getData(applicationIdPath()).map(d -> ApplicationId.fromSerializedForm(Utf8.toString(d)));
}
void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) {
applicationPackageReference.ifPresent(
- reference -> configCurator.putData(applicationPackageReferencePath(), reference.value()));
+ reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value())));
}
FileReference readApplicationPackageReference() {
- if ( ! configCurator.exists(applicationPackageReferencePath())) return null; // This should not happen.
- return new FileReference(configCurator.getData(applicationPackageReferencePath()));
+ Optional<byte[]> data = curator.getData(applicationPackageReferencePath());
+ if (data.isEmpty()) return null; // This should not happen.
+
+ return new FileReference(Utf8.toString(data.get()));
}
- private String applicationPackageReferencePath() {
- return sessionPath.append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute();
+ private Path applicationPackageReferencePath() {
+ return sessionPath.append(APPLICATION_PACKAGE_REFERENCE_PATH);
}
- private String versionPath() {
- return sessionPath.append(VERSION_PATH).getAbsolute();
+ private Path versionPath() {
+ return sessionPath.append(VERSION_PATH);
}
- private String dockerImageRepositoryPath() {
- return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute();
+ private Path dockerImageRepositoryPath() {
+ return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH);
}
- private String athenzDomainPath() {
- return sessionPath.append(ATHENZ_DOMAIN).getAbsolute();
+ private Path athenzDomainPath() {
+ return sessionPath.append(ATHENZ_DOMAIN);
}
- private String quotaPath() {
- return sessionPath.append(QUOTA_PATH).getAbsolute();
+ private Path quotaPath() {
+ return sessionPath.append(QUOTA_PATH);
}
- private String tenantSecretStorePath() {
- return sessionPath.append(TENANT_SECRET_STORES_PATH).getAbsolute();
+ private Path tenantSecretStorePath() {
+ return sessionPath.append(TENANT_SECRET_STORES_PATH);
}
- private String operatorCertificatesPath() {
- return sessionPath.append(OPERATOR_CERTIFICATES_PATH).getAbsolute();
+ private Path operatorCertificatesPath() {
+ return sessionPath.append(OPERATOR_CERTIFICATES_PATH);
}
public void writeVespaVersion(Version version) {
- configCurator.putData(versionPath(), version.toString());
+ curator.set(versionPath(), Utf8.toBytes(version.toString()));
}
public Version readVespaVersion() {
- if ( ! configCurator.exists(versionPath())) return Vtag.currentVersion; // TODO: This should not be possible any more - verify and remove
- return new Version(configCurator.getData(versionPath()));
+ Optional<byte[]> data = curator.getData(versionPath());
+ // TODO: Empty version should not be possible any more - verify and remove
+ return data.map(d -> new Version(Utf8.toString(d))).orElse(Vtag.currentVersion);
}
public Optional<DockerImage> readDockerImageRepository() {
- if ( ! configCurator.exists(dockerImageRepositoryPath())) return Optional.empty();
- String dockerImageRepository = configCurator.getData(dockerImageRepositoryPath());
- return dockerImageRepository.isEmpty() ? Optional.empty() : Optional.of(DockerImage.fromString(dockerImageRepository));
+ Optional<byte[]> dockerImageRepository = curator.getData(dockerImageRepositoryPath());
+ return dockerImageRepository.map(d -> DockerImage.fromString(Utf8.toString(d)));
}
public void writeDockerImageRepository(Optional<DockerImage> dockerImageRepository) {
- dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo.untagged()));
+ dockerImageRepository.ifPresent(repo -> curator.set(dockerImageRepositoryPath(), Utf8.toBytes(repo.untagged())));
}
public Instant readCreateTime() {
- String path = getCreateTimePath();
- if ( ! configCurator.exists(path)) return Instant.EPOCH;
- return Instant.ofEpochSecond(Long.parseLong(configCurator.getData(path)));
+ Optional<byte[]> data = curator.getData(getCreateTimePath());
+ return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))).orElse(Instant.EPOCH);
}
- private String getCreateTimePath() {
- return sessionPath.append(CREATE_TIME_PATH).getAbsolute();
+ private Path getCreateTimePath() {
+ return sessionPath.append(CREATE_TIME_PATH);
}
AllocatedHosts getAllocatedHosts() {
@@ -232,14 +233,13 @@ public class SessionZooKeeperClient {
}
public ZooKeeperDeployer createDeployer(DeployLogger logger) {
- ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, sessionPath);
+ ZooKeeperClient zkClient = new ZooKeeperClient(curator, logger, sessionPath);
return new ZooKeeperDeployer(zkClient);
}
public Transaction createWriteStatusTransaction(Session.Status status) {
- String path = sessionStatusPath.getAbsolute();
CuratorTransaction transaction = new CuratorTransaction(curator);
- if (configCurator.exists(path)) {
+ if (curator.exists(sessionStatusPath)) {
transaction.add(CuratorOperations.setData(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name())));
} else {
transaction.add(CuratorOperations.create(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name())));
@@ -248,59 +248,56 @@ public class SessionZooKeeperClient {
}
public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) {
- athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.value()));
+ athenzDomain.ifPresent(domain -> curator.set(athenzDomainPath(), Utf8.toBytes(domain.value())));
}
public Optional<AthenzDomain> readAthenzDomain() {
- if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty();
- return Optional.ofNullable(configCurator.getData(athenzDomainPath()))
- .filter(domain -> ! domain.isBlank())
- .map(AthenzDomain::from);
+ return curator.getData(athenzDomainPath())
+ .map(Utf8::toString)
+ .filter(domain -> !domain.isBlank())
+ .map(AthenzDomain::from);
}
public void writeQuota(Optional<Quota> maybeQuota) {
maybeQuota.ifPresent(quota -> {
var bytes = uncheck(() -> SlimeUtils.toJsonBytes(quota.toSlime()));
- configCurator.putData(quotaPath(), bytes);
+ curator.set(quotaPath(), bytes);
});
}
public Optional<Quota> readQuota() {
- if ( ! configCurator.exists(quotaPath())) return Optional.empty();
- return Optional.ofNullable(configCurator.getData(quotaPath()))
- .map(SlimeUtils::jsonToSlime)
- .map(slime -> Quota.fromSlime(slime.get()));
+ return curator.getData(quotaPath())
+ .map(SlimeUtils::jsonToSlime)
+ .map(slime -> Quota.fromSlime(slime.get()));
}
public void writeTenantSecretStores(List<TenantSecretStore> tenantSecretStores) {
if (!tenantSecretStores.isEmpty()) {
var bytes = uncheck(() -> SlimeUtils.toJsonBytes(TenantSecretStoreSerializer.toSlime(tenantSecretStores)));
- configCurator.putData(tenantSecretStorePath(), bytes);
+ curator.set(tenantSecretStorePath(), bytes);
}
}
public List<TenantSecretStore> readTenantSecretStores() {
- if ( ! configCurator.exists(tenantSecretStorePath())) return List.of();
- return Optional.ofNullable(configCurator.getData(tenantSecretStorePath()))
- .map(SlimeUtils::jsonToSlime)
- .map(slime -> TenantSecretStoreSerializer.listFromSlime(slime.get()))
- .orElse(List.of());
+ return curator.getData(tenantSecretStorePath())
+ .map(SlimeUtils::jsonToSlime)
+ .map(slime -> TenantSecretStoreSerializer.listFromSlime(slime.get()))
+ .orElse(List.of());
}
public void writeOperatorCertificates(List<X509Certificate> certificates) {
if( ! certificates.isEmpty()) {
var bytes = uncheck(() -> SlimeUtils.toJsonBytes(OperatorCertificateSerializer.toSlime(certificates)));
- configCurator.putData(operatorCertificatesPath(), bytes);
+ curator.set(operatorCertificatesPath(), bytes);
}
}
public List<X509Certificate> readOperatorCertificates() {
- if ( ! configCurator.exists(operatorCertificatesPath())) return List.of();
- return Optional.ofNullable(configCurator.getData(operatorCertificatesPath()))
- .map(SlimeUtils::jsonToSlime)
- .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get()))
- .orElse(List.of());
+ return curator.getData(operatorCertificatesPath())
+ .map(SlimeUtils::jsonToSlime)
+ .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get()))
+ .orElse(List.of());
}
/**
@@ -313,7 +310,7 @@ public class SessionZooKeeperClient {
transaction.add(CuratorOperations.create(sessionPath.getAbsolute()));
transaction.add(CuratorOperations.create(sessionPath.append(UPLOAD_BARRIER).getAbsolute()));
transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations());
- transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond()))));
+ transaction.add(CuratorOperations.create(getCreateTimePath().getAbsolute(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond()))));
transaction.commit();
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
index 2f7b397cbd9..b10a90de714 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.concurrent.Lock;
import com.yahoo.concurrent.Locks;
@@ -30,7 +31,6 @@ import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
import com.yahoo.vespa.config.server.session.SessionPreparer;
import com.yahoo.vespa.config.server.session.SessionRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.transaction.CuratorOperations;
import com.yahoo.vespa.curator.transaction.CuratorTransaction;
@@ -98,7 +98,6 @@ public class TenantRepository {
private final Locks<TenantName> tenantLocks = new Locks<>(1, TimeUnit.MINUTES);
private final HostRegistry hostRegistry;
private final TenantListener tenantListener;
- private final ConfigCurator configCurator;
private final Curator curator;
private final Metrics metrics;
private final MetricUpdater metricUpdater;
@@ -120,13 +119,14 @@ public class TenantRepository {
private final ScheduledExecutorService checkForRemovedApplicationsService =
new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications"));
private final Optional<Curator.DirectoryCache> directoryCache;
+ private final ZookeeperServerConfig zookeeperServerConfig;
/**
* Creates a new tenant repository
*/
@Inject
public TenantRepository(HostRegistry hostRegistry,
- ConfigCurator configCurator,
+ Curator curator,
Metrics metrics,
FlagSource flagSource,
SecretStore secretStore,
@@ -137,9 +137,10 @@ public class TenantRepository {
ModelFactoryRegistry modelFactoryRegistry,
ConfigDefinitionRepo configDefinitionRepo,
ReloadListener reloadListener,
- TenantListener tenantListener) {
+ TenantListener tenantListener,
+ ZookeeperServerConfig zookeeperServerConfig) {
this(hostRegistry,
- configCurator,
+ curator,
metrics,
new StripedExecutor<>(),
new StripedExecutor<>(),
@@ -155,11 +156,12 @@ public class TenantRepository {
modelFactoryRegistry,
configDefinitionRepo,
reloadListener,
- tenantListener);
+ tenantListener,
+ zookeeperServerConfig);
}
public TenantRepository(HostRegistry hostRegistry,
- ConfigCurator configCurator,
+ Curator curator,
Metrics metrics,
StripedExecutor<TenantName> zkApplicationWatcherExecutor ,
StripedExecutor<TenantName> zkSessionWatcherExecutor,
@@ -175,12 +177,13 @@ public class TenantRepository {
ModelFactoryRegistry modelFactoryRegistry,
ConfigDefinitionRepo configDefinitionRepo,
ReloadListener reloadListener,
- TenantListener tenantListener) {
+ TenantListener tenantListener,
+ ZookeeperServerConfig zookeeperServerConfig) {
this.hostRegistry = hostRegistry;
this.configserverConfig = configserverConfig;
this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(),
new DaemonThreadFactory("bootstrap-tenant-"));
- this.curator = configCurator.curator();
+ this.curator = curator;
this.metrics = metrics;
metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap());
this.zkCacheExecutor = zkCacheExecutor;
@@ -197,7 +200,7 @@ public class TenantRepository {
this.configDefinitionRepo = configDefinitionRepo;
this.reloadListener = reloadListener;
this.tenantListener = tenantListener;
- this.configCurator = configCurator;
+ this.zookeeperServerConfig = zookeeperServerConfig;
curator.framework().getConnectionStateListenable().addListener(this::stateChanged);
@@ -344,7 +347,7 @@ public class TenantRepository {
SessionRepository sessionRepository = new SessionRepository(tenantName,
applicationRepo,
sessionPreparer,
- configCurator,
+ curator,
metrics,
zkSessionWatcherExecutor,
permanentApplicationPackage,
@@ -357,7 +360,8 @@ public class TenantRepository {
zone,
clock,
modelFactoryRegistry,
- configDefinitionRepo);
+ configDefinitionRepo,
+ zookeeperServerConfig.juteMaxBuffer());
log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created +
". Bootstrapping in " + Duration.between(start, Instant.now()));
Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created);
@@ -528,7 +532,7 @@ public class TenantRepository {
}
public void close() {
- directoryCache.ifPresent(Curator.DirectoryCache::close);
+ directoryCache.ifPresent(com.yahoo.vespa.curator.Curator.DirectoryCache::close);
try {
zkCacheExecutor.shutdown();
checkForRemovedApplicationsService.shutdown();
@@ -601,6 +605,6 @@ public class TenantRepository {
return barriersPath;
}
- public Curator getCurator() { return curator; }
+ public com.yahoo.vespa.curator.Curator getCurator() { return curator; }
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java
deleted file mode 100644
index b13750f93d4..00000000000
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java
+++ /dev/null
@@ -1,255 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.zookeeper;
-
-import com.google.inject.Inject;
-import com.yahoo.cloud.config.ZookeeperServerConfig;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.curator.Curator;
-import org.apache.zookeeper.KeeperException;
-
-import java.util.List;
-import java.util.logging.Level;
-
-/**
- * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server
- * and knows about how config content is mapped to node names and stored.
- * <p>
- * Usage details:
- * Config ids are stored as foo#bar#c0 instead of foo/bar/c0, for simplicity.
- * Keep the amount of domain-specific logic here to a minimum.
- * Data for one application x is stored on this form:
- * /config/v2/tenants/x/sessions/y/defconfigs
- * /config/v2/tenants/x/sessions/y/userapp
- * <p>
- * The user application structure is exactly the same as in the user's app dir during deploy.
- * The current live app id (for example y) is stored in the node //config/v2/tenants/x/applications/&lt;application-id&gt;
- * It is updated outside this class, typically in config server when activating config
- *
- * @author Vegard Havdal
- * @author bratseth
- */
-public class ConfigCurator {
-
- /** Path for def files, under one app */
- public static final String DEFCONFIGS_ZK_SUBPATH = "/defconfigs";
-
- /** Path for def files, under one app */
- public static final String USER_DEFCONFIGS_ZK_SUBPATH = "/userdefconfigs";
-
- /** Path for metadata about an application */
- public static final String META_ZK_PATH = "/meta";
-
- /** Path for the app package's dir structure, under one app */
- public static final String USERAPP_ZK_SUBPATH = "/userapp";
-
- /** Path for session state */
- public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState";
-
- private final Curator curator;
-
- public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName());
-
- /** The maximum size of a ZooKeeper node */
- private final int maxNodeSize;
-
- public static ConfigCurator create(Curator curator) {
- return new ConfigCurator(curator, 1024*1024*10);
- }
-
- @Inject
- public ConfigCurator(Curator curator, ZookeeperServerConfig config) {
- this(curator, config.juteMaxBuffer());
- }
-
- private ConfigCurator(Curator curator, int maxNodeSize) {
- this.curator = curator;
- this.maxNodeSize = maxNodeSize;
- log.log(Level.CONFIG, "Using jute max buffer size " + this.maxNodeSize);
- testZkConnection();
- }
-
- /** Returns the curator instance this wraps */
- public Curator curator() { return curator; }
-
- /** Cleans and creates a zookeeper completely */
- void initAndClear(String path) {
- try {
- if (exists(path))
- deleteRecurse(path);
- createRecurse(path);
- }
- catch (Exception e) {
- throw new RuntimeException("Exception clearing path " + path + " in ZooKeeper", e);
- }
- }
-
- /** Creates a path. If the path already exists this does nothing. */
- private void createRecurse(String path) {
- try {
- if (exists(path)) return;
- curator.framework().create().creatingParentsIfNeeded().forPath(path);
- }
- catch (KeeperException.NodeExistsException e) {
- // Ignore, path already exists
- }
- catch(Exception e){
- throw new RuntimeException("Exception creating path " + path + " in ZooKeeper", e);
- }
- }
-
- /** Returns the data at a path and node. Replaces / by # in node names. */
- public String getData(String path, String node) {
- return getData(createFullPath(path, node));
- }
-
- /** Returns the data at a path */
- public String getData(String path) {
- byte[] data = getBytes(path);
- return (data == null) ? null : Utf8.toString(data);
- }
-
- /**
- * Returns the data at a path, or null if the path does not exist.
- *
- * @param path a String with a pathname.
- * @return a byte array with data.
- */
- public byte[] getBytes(String path) {
- if ( ! exists(path)) throw new IllegalArgumentException("Cannot read data from path " + path + ", it does not exist");
-
- try {
- return curator.framework().getData().forPath(path);
- }
- catch (Exception e) {
- throw new RuntimeException("Exception reading from path " + path + " in ZooKeeper", e);
- }
- }
-
- /** Returns whether a path exists in zookeeper */
- public boolean exists(String path, String node) {
- return exists(createFullPath(path, node));
- }
-
- /** Returns whether a path exists in zookeeper */
- public boolean exists(String path) {
- try {
- return curator.framework().checkExists().forPath(path) != null;
- }
- catch (Exception e) {
- throw new RuntimeException("Exception checking existence of path " + path + " in ZooKeeper", e);
- }
- }
-
- /** Creates a Zookeeper node. If the node already exists this does nothing. */
- public void createNode(String path) {
- if ( ! exists(path))
- createRecurse(path);
- }
-
- /** Creates a Zookeeper node synchronously. Replaces / by # in node names. */
- public void createNode(String path, String node) {
- createNode(createFullPath(path, node));
- }
-
- private String createFullPath(String path, String node) {
- return path + "/" + toConfigserverName(node);
- }
-
- /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */
- public void putData(String path, String node, String data) {
- putData(path, node, Utf8.toBytes(data));
- }
-
- /** Sets data at a given path. Creates the node if it doesn't exist */
- public void putData(String path, String data) {
- putData(path, Utf8.toBytes(data));
- }
-
- private void ensureDataIsNotTooLarge(byte[] toPut, String path) {
- if (toPut.length >= maxNodeSize) {
- throw new IllegalArgumentException("Error: too much zookeeper data in node: "
- + "[" + toPut.length + " bytes] (path " + path + ")");
- }
- }
-
- /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */
- private void putData(String path, String node, byte[] data) {
- putData(createFullPath(path, node), data);
- }
-
- /** Sets data at a given path. Creates the path if it doesn't exist */
- public void putData(String path, byte[] data) {
- try {
- ensureDataIsNotTooLarge(data, path);
- if (exists(path))
- curator.framework().setData().forPath(path, data);
- else
- curator.framework().create().creatingParentsIfNeeded().forPath(path, data);
- }
- catch (Exception e) {
- throw new RuntimeException("Exception writing to path " + path + " in ZooKeeper", e);
- }
- }
-
- /**
- * Replaces / with # in the given node.
- *
- * @param node a zookeeper node name
- * @return a config server node name
- */
- private String toConfigserverName(String node) {
- if (node.startsWith("/")) node = node.substring(1);
- return node.replaceAll("/", "#");
- }
-
- /**
- * Lists thh children at the given path.
- *
- * @return the local names of the children at this path, or an empty list (never null) if none.
- */
- public List<String> getChildren(String path) {
- try {
- return curator.framework().getChildren().forPath(path);
- }
- catch (Exception e) {
- throw new RuntimeException("Exception getting children of path " + path + " in ZooKeeper", e);
- }
- }
-
- /**
- * Puts config definition data and metadata into ZK.
- *
- * @param name The config definition name (including namespace)
- * @param path /zoopath
- * @param data The contents to write to ZK (as a byte array)
- */
- public void putDefData(String name, String path, byte[] data) {
- putData(path, name, data);
- }
-
- /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */
- public void deleteRecurse(String path) {
- try {
- if ( ! exists(path)) return;
- curator.framework().delete().deletingChildrenIfNeeded().forPath(path);
- }
- catch (Exception e) {
- throw new RuntimeException("Exception deleting path " + path, e);
- }
- }
-
- private void testZkConnection() { // This is not necessary, but allows us to give a useful error message
- if (curator.connectionSpec().isEmpty()) return;
- try {
- curator.framework().checkExists().forPath("/dummy");
- }
- catch (Exception e) {
- log.log(Level.SEVERE, "Unable to connect to ZooKeeper on " + curator.connectionSpec() +
- ". Please verify that VESPA_CONFIGSERVERS points to the correct configserver(s) " +
- "on all config server nodes and are the same config server(s) as in services.xml, " +
- "and that they are started. " +
- "Check the log(s) for config server errors. Aborting.", e);
- }
- }
-
-}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java
index 216c9edd0d0..f36ec164a14 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java
@@ -2,11 +2,15 @@
package com.yahoo.vespa.config.server.zookeeper;
import java.util.logging.Level;
+
+import com.yahoo.path.Path;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.recipes.CuratorCounter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.logging.Logger;
/**
* A counter that sets its initial value to the number of apps in zookeeper if no counter value is set. Subclass
@@ -16,14 +20,14 @@ import java.util.List;
*/
public class InitializedCounter {
- private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(InitializedCounter.class.getName());
+ private static final Logger log = java.util.logging.Logger.getLogger(InitializedCounter.class.getName());
final CuratorCounter counter;
- private final String sessionsDirPath;
+ private final Path sessionsDirPath;
- InitializedCounter(ConfigCurator configCurator, String counterPath, String sessionsDirPath) {
+ InitializedCounter(Curator curator, Path counterPath, Path sessionsDirPath) {
this.sessionsDirPath = sessionsDirPath;
- this.counter = new CuratorCounter(configCurator.curator(), counterPath);
- initializeCounterValue(getLatestSessionId(configCurator, sessionsDirPath));
+ this.counter = new CuratorCounter(curator, counterPath);
+ initializeCounterValue(getLatestSessionId(curator, sessionsDirPath));
}
private void initializeCounterValue(Long latestSessionId) {
@@ -40,8 +44,8 @@ public class InitializedCounter {
*
* @return true, if an application exists, false otherwise
*/
- private static boolean applicationExists(ConfigCurator configCurator, String appsPath) {
- return configCurator.exists(appsPath);
+ private static boolean applicationExists(Curator curator, Path appsPath) {
+ return curator.exists(appsPath);
}
/**
@@ -50,12 +54,12 @@ public class InitializedCounter {
*
* @return generation of the latest deployed application
*/
- private static Long getLatestSessionId(ConfigCurator configCurator, String appsPath) {
- if (!applicationExists(configCurator, appsPath)) return null;
+ private static Long getLatestSessionId(Curator curator, Path appsPath) {
+ if (!applicationExists(curator, appsPath)) return null;
Long newestGeneration = null;
try {
- if (!getDeployedApplicationGenerations(configCurator, appsPath).isEmpty()) {
- newestGeneration = Collections.max(getDeployedApplicationGenerations(configCurator, appsPath));
+ if (!getDeployedApplicationGenerations(curator, appsPath).isEmpty()) {
+ newestGeneration = Collections.max(getDeployedApplicationGenerations(curator, appsPath));
}
} catch (Exception e) {
log.log(Level.WARNING, "Could not get newest application generation from Zookeeper");
@@ -63,10 +67,10 @@ public class InitializedCounter {
return newestGeneration;
}
- private static List<Long> getDeployedApplicationGenerations(ConfigCurator configCurator, String appsPath) {
+ private static List<Long> getDeployedApplicationGenerations(Curator curator, Path appsPath) {
ArrayList<Long> generations = new ArrayList<>();
try {
- List<String> stringGenerations = configCurator.getChildren(appsPath);
+ List<String> stringGenerations = curator.getChildren(appsPath);
if (stringGenerations != null && !(stringGenerations.isEmpty())) {
for (String s : stringGenerations) {
generations.add(Long.parseLong(s));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java
index 17d0f7e426e..c30743bbaea 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.zookeeper;
import com.yahoo.config.provision.TenantName;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import com.yahoo.vespa.curator.Curator;
/**
* A counter keeping track of session ids in an atomic fashion across multiple config servers.
@@ -11,10 +12,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
*/
public class SessionCounter extends InitializedCounter {
- public SessionCounter(ConfigCurator configCurator, TenantName tenantName) {
- super(configCurator,
- TenantRepository.getTenantPath(tenantName).append("sessionCounter").getAbsolute(),
- TenantRepository.getSessionsPath(tenantName).getAbsolute());
+ public SessionCounter(Curator curator, TenantName tenantName) {
+ super(curator,
+ TenantRepository.getTenantPath(tenantName).append("sessionCounter"),
+ TenantRepository.getSessionsPath(tenantName));
}
/**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
index 084f26bd368..04ce5b4f63a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
@@ -3,11 +3,12 @@ package com.yahoo.vespa.config.server.zookeeper;
import com.yahoo.io.reader.NamedReader;
import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.curator.Curator;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
/**
@@ -18,12 +19,29 @@ import java.util.List;
*/
public class ZKApplication {
- private final ConfigCurator zk;
+ /** Path for def files, under one app */
+ public static final String DEFCONFIGS_ZK_SUBPATH = "/defconfigs";
+ /** Path for def files, under one app */
+ public static final String USER_DEFCONFIGS_ZK_SUBPATH = "/userdefconfigs";
+ /** Path for metadata about an application */
+ public static final String META_ZK_PATH = "/meta";
+ /** Path for the app package's dir structure, under one app */
+ public static final String USERAPP_ZK_SUBPATH = "/userapp";
+ /** Path for session state */
+ public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState";
+ private final Curator curator;
private final Path appPath;
+ /** The maximum size of a ZooKeeper node */
+ private final int maxNodeSize;
- ZKApplication(ConfigCurator zk, Path appPath) {
- this.zk = zk;
+ ZKApplication(Curator curator, Path appPath, int maxNodeSize) {
+ this.curator = curator;
this.appPath = appPath;
+ this.maxNodeSize = maxNodeSize;
+ }
+
+ ZKApplication(Curator curator, Path appPath) {
+ this(curator, appPath, 10 * 1024 * 1024);
}
/**
@@ -37,7 +55,7 @@ public class ZKApplication {
* @return the files in the given path, or an empty list if the directory does not exist or is empty.
* The list gets owned by the caller and can be modified freely.
*/
- List<NamedReader> getAllDataFromDirectory(String path, String fileNameSuffix, boolean recursive) {
+ List<NamedReader> getAllDataFromDirectory(Path path, String fileNameSuffix, boolean recursive) {
return getAllDataFromDirectory(path, "", fileNameSuffix, recursive);
}
@@ -46,23 +64,22 @@ public class ZKApplication {
*
* @param namePrefix the prefix to prepend to the returned reader names
*/
- private List<NamedReader> getAllDataFromDirectory(String path, String namePrefix, String fileNameSuffix, boolean recursive) {
- String fullPath = getFullPath(path);
+ private List<NamedReader> getAllDataFromDirectory(Path path, String namePrefix, String fileNameSuffix, boolean recursive) {
List<NamedReader> result = new ArrayList<>();
List<String> children = getChildren(path);
try {
for (String child : children) {
if (fileNameSuffix == null || child.endsWith(fileNameSuffix)) {
- result.add(new NamedReader(namePrefix + child, reader(zk.getData(fullPath, child))));
+ result.add(new NamedReader(namePrefix + child, reader(getData(path.append(child)))));
}
if (recursive)
- result.addAll(getAllDataFromDirectory(path + "/" + child,
+ result.addAll(getAllDataFromDirectory(path.append(child),
namePrefix + child + "/", fileNameSuffix, recursive));
}
return result;
} catch (Exception e) {
- throw new RuntimeException("Could not retrieve all data from '" + fullPath + "' in zookeeper", e);
+ throw new RuntimeException("Could not retrieve all data from '" + path + "' in zookeeper", e);
}
}
@@ -70,59 +87,45 @@ public class ZKApplication {
* Retrieves a node relative to the node of the live application
*
* @param path a path relative to the currently active application
- * @param node a path relative to the path above
* @return a Reader that can be used to get the data
*/
- Reader getDataReader(String path, String node) {
- return reader(getData(path, node));
+ Reader getDataReader(Path path) {
+ return reader(getData(path));
}
- public String getData(String path, String node) {
- if ( ! exists(path, node)) throw new IllegalArgumentException("No node for " + getFullPath(path) + "/" + node + " exists");
-
- try {
- return zk.getData(getFullPath(path), node);
- } catch (Exception e) {
- throw new IllegalArgumentException("Could not retrieve node '" +
- getFullPath(path) + "/" + node + "' in zookeeper", e);
- }
+ NamedReader getNamedReader(String name, Path path) {
+ return new NamedReader(name, reader(getData(path)));
}
- public String getData(String path) {
- if ( ! exists(path)) throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists");
+ public String getData(Path path) {
+ return Utf8.toString(getBytesInternal(getFullPath(path)));
+ }
- try {
- return zk.getData(getFullPath(path));
- } catch (RuntimeException e) {
- throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e);
- }
+ private byte[] getBytesInternal(Path path) {
+ return curator.getData(path)
+ .orElseThrow(() -> new IllegalArgumentException("Could not get data from '" +
+ path + "' in zookeeper"));
}
- public byte[] getBytes(String path) {
- try {
- return zk.getBytes(getFullPath(path));
- } catch (RuntimeException e) {
- throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e);
- }
+ public byte[] getBytes(Path path) {
+ return getBytesInternal(getFullPath(path));
}
- void putData(String path, String data) {
+ void putData(Path path, String data) {
+ byte[] bytes = Utf8.toBytes(data);
+ ensureDataIsNotTooLarge(bytes, path);
try {
- zk.putData(getFullPath(path), data);
+ curator.set(getFullPath(path), bytes);
} catch (RuntimeException e) {
throw new IllegalArgumentException("Could not put data to node '" + getFullPath(path) + "' in zookeeper", e);
}
}
- /**
- * Checks if the given node exists under path under this live app
- *
- * @param path a zookeeper path
- * @param node a zookeeper node
- * @return true if the node exists in the path, false otherwise
- */
- public boolean exists(String path, String node) {
- return zk.exists(getFullPath(path), node);
+ private void ensureDataIsNotTooLarge(byte[] toPut, Path path) {
+ if (toPut.length >= maxNodeSize) {
+ throw new IllegalArgumentException("Error: too much zookeeper data in node: "
+ + "[" + toPut.length + " bytes] (path " + path + ")");
+ }
}
/**
@@ -131,16 +134,16 @@ public class ZKApplication {
* @param path a zookeeper path
* @return true if the node exists in the path, false otherwise
*/
- public boolean exists(String path) {
- return zk.exists(getFullPath(path));
+ public boolean exists(Path path) {
+ return curator.exists(getFullPath(path));
}
- private String getFullPath(String path) {
+ private Path getFullPath(Path path) {
Path fullPath = appPath;
if (path != null) {
fullPath = appPath.append(path);
}
- return fullPath.getAbsolute();
+ return fullPath;
}
/**
@@ -148,8 +151,8 @@ public class ZKApplication {
*
* @param path path to delete
*/
- void deleteRecurse(String path) {
- zk.deleteRecurse(getFullPath(path));
+ void deleteRecurse(Path path) {
+ curator.delete(getFullPath(path));
}
/**
@@ -158,31 +161,21 @@ public class ZKApplication {
* @param path a path relative to the currently active application
* @return a list of file names, which is empty (never null) if the path does not exist
*/
- public List<String> getChildren(String path) {
- String fullPath = getFullPath(path);
- if (! zk.exists(fullPath)) return Collections.emptyList();
- return zk.getChildren(fullPath);
+ public List<String> getChildren(Path path) {
+ return curator.getChildren(getFullPath(path));
}
private static Reader reader(String string) {
return new StringReader(string);
}
- public void create(String path) {
- if (path != null && !path.startsWith("/")) path = "/" + path;
+ public void create(Path path) {
try {
- zk.createNode(getFullPath(path));
+ curator.create(getFullPath(path));
} catch (RuntimeException e) {
throw new IllegalArgumentException(e);
}
}
- Reader getDataReader(String path) {
- String data = getData(path);
- if (data == null)
- throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists");
- return reader(data);
- }
-
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
index 674c0f72c40..3b828e32ed7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
@@ -5,14 +5,23 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.path.Path;
import com.yahoo.io.IOUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.util.logging.Level;
import com.yahoo.vespa.config.util.ConfigUtils;
-import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+
/**
* @author Ulf Lilleengen
* @author Vegard Havdal
@@ -30,7 +39,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public boolean isDirectory() {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
if (zkApp.exists(zkPath)) {
String data = zkApp.getData(zkPath);
return data == null || data.isEmpty() || ! zkApp.getChildren(zkPath).isEmpty();
@@ -41,7 +50,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public boolean exists() {
try {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
return zkApp.exists(zkPath);
} catch (RuntimeException e) {
return false;
@@ -60,7 +69,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public Reader createReader() throws FileNotFoundException {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
if ( ! zkApp.exists(zkPath)) throw new FileNotFoundException("No such path: " + path);
return new StringReader(zkApp.getData(zkPath));
@@ -68,7 +77,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public InputStream createInputStream() throws FileNotFoundException {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
if ( ! zkApp.exists(zkPath)) throw new FileNotFoundException("No such path: " + path);
return new ByteArrayInputStream(zkApp.getBytes(zkPath));
@@ -76,7 +85,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public ApplicationFile createDirectory() {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
if (isDirectory()) return this;
if (exists()) {
throw new IllegalArgumentException("Unable to create directory, file exists: " + path);
@@ -88,7 +97,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public ApplicationFile writeFile(Reader input) {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
try {
String data = IOUtils.readAll(input);
String status = ContentStatusNew;
@@ -105,7 +114,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public ApplicationFile appendFile(String value) {
- String zkPath = getZKPath(path);
+ Path zkPath = getZKPath(path);
String status = ContentStatusNew;
if (zkApp.exists(zkPath)) {
status = ContentStatusChanged;
@@ -120,7 +129,7 @@ class ZKApplicationFile extends ApplicationFile {
@Override
public List<ApplicationFile> listFiles(PathFilter filter) {
- String userPath = getZKPath(path);
+ Path userPath = getZKPath(path);
List<ApplicationFile> ret = new ArrayList<>();
for (String zkChild : zkApp.getChildren(userPath)) {
Path childPath = path.append(zkChild);
@@ -132,15 +141,15 @@ class ZKApplicationFile extends ApplicationFile {
return ret;
}
- private static String getZKPath(Path path) {
+ private static Path getZKPath(Path path) {
if (path.isRoot()) {
- return ConfigCurator.USERAPP_ZK_SUBPATH;
+ return Path.fromString(USERAPP_ZK_SUBPATH);
}
- return ConfigCurator.USERAPP_ZK_SUBPATH + "/" + path.getRelative();
+ return Path.fromString(USERAPP_ZK_SUBPATH).append(path);
}
private void writeMetaFile(String input, String status) {
- String metaPath = getZKPath(getMetaPath());
+ Path metaPath = getZKPath(getMetaPath());
StringWriter writer = new StringWriter();
try {
mapper.writeValue(writer, new MetaData(status, input == null ? "" : ConfigUtils.getMd5(input)));
@@ -152,7 +161,7 @@ class ZKApplicationFile extends ApplicationFile {
}
public MetaData getMetaData() {
- String metaPath = getZKPath(getMetaPath());
+ Path metaPath = getZKPath(getMetaPath());
log.log(Level.FINE, () -> "Getting metadata for " + metaPath);
if (!zkApp.exists(getZKPath(path))) {
if (zkApp.exists(metaPath)) {
@@ -167,7 +176,7 @@ class ZKApplicationFile extends ApplicationFile {
return new MetaData(ContentStatusNew, isDirectory() ? "" : ConfigUtils.getMd5(zkApp.getData(getZKPath(path))));
}
- private MetaData getMetaDataFromZk(String metaPath) {
+ private MetaData getMetaDataFromZk(Path metaPath) {
try {
return mapper.readValue(zkApp.getBytes(metaPath), MetaData.class);
} catch (IOException e) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index bcb19a8f25a..cc9b799f2d9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -21,10 +21,10 @@ import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigDefinitionBuilder;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.util.ConfigUtils;
+import com.yahoo.vespa.curator.Curator;
import java.io.File;
import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -33,6 +33,9 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+
/**
* Represents an application residing in zookeeper.
*
@@ -49,16 +52,21 @@ public class ZKApplicationPackage implements ApplicationPackage {
public static final String allocatedHostsNode = "allocatedHosts";
private final ApplicationMetaData metaData;
- public ZKApplicationPackage(ConfigCurator zk, Path sessionPath) {
- verifyAppPath(zk, sessionPath);
- zkApplication = new ZKApplication(zk, sessionPath);
+ public ZKApplicationPackage(Curator curator, Path sessionPath, int maxNodeSize) {
+ verifyAppPath(curator, sessionPath);
+ zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize);
metaData = readMetaDataFromLiveApp(zkApplication);
importFileRegistries();
allocatedHosts = importAllocatedHosts();
}
+ // For testing
+ ZKApplicationPackage(Curator curator, Path sessionPath) {
+ this(curator, sessionPath, 10 * 1024 * 1024);
+ }
+
private Optional<AllocatedHosts> importAllocatedHosts() {
- if ( ! zkApplication.exists(allocatedHostsNode)) return Optional.empty();
+ if ( ! zkApplication.exists(Path.fromString(allocatedHostsNode))) return Optional.empty();
return Optional.of(readAllocatedHosts());
}
@@ -69,14 +77,14 @@ public class ZKApplicationPackage implements ApplicationPackage {
*/
private AllocatedHosts readAllocatedHosts() {
try {
- return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(allocatedHostsNode));
+ return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(Path.fromString(allocatedHostsNode)));
} catch (Exception e) {
throw new RuntimeException("Unable to read allocated hosts", e);
}
}
private void importFileRegistries() {
- List<String> perVersionFileRegistryNodes = zkApplication.getChildren(fileRegistryNode);
+ List<String> perVersionFileRegistryNodes = zkApplication.getChildren(Path.fromString(fileRegistryNode));
perVersionFileRegistryNodes
.forEach(version ->
fileRegistryMap.put(Version.fromString(version),
@@ -85,18 +93,19 @@ public class ZKApplicationPackage implements ApplicationPackage {
private PreGeneratedFileRegistry importFileRegistry(String fileRegistryNode) {
try {
- return PreGeneratedFileRegistry.importRegistry(zkApplication.getDataReader(fileRegistryNode));
+ return PreGeneratedFileRegistry.importRegistry(zkApplication.getDataReader(Path.fromString(fileRegistryNode)));
} catch (Exception e) {
throw new RuntimeException("Could not determine which files to distribute", e);
}
}
private ApplicationMetaData readMetaDataFromLiveApp(ZKApplication liveApp) {
- String metaDataString = liveApp.getData(ConfigCurator.META_ZK_PATH);
+ Path metaPath = Path.fromString(ZKApplication.META_ZK_PATH);
+ String metaDataString = liveApp.getData(metaPath);
if (metaDataString == null || metaDataString.isEmpty()) {
return null;
}
- return ApplicationMetaData.fromJsonString(liveApp.getData(ConfigCurator.META_ZK_PATH));
+ return ApplicationMetaData.fromJsonString(liveApp.getData(metaPath));
}
@Override
@@ -104,8 +113,8 @@ public class ZKApplicationPackage implements ApplicationPackage {
return metaData;
}
- private static void verifyAppPath(ConfigCurator zk, Path appPath) {
- if (!zk.exists(appPath.getAbsolute()))
+ private static void verifyAppPath(Curator zk, Path appPath) {
+ if (!zk.exists(appPath))
throw new RuntimeException("App with path " + appPath + " does not exist");
}
@@ -125,7 +134,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public Reader getHosts() {
- if (zkApplication.exists(ConfigCurator.USERAPP_ZK_SUBPATH, HOSTS))
+ if (zkApplication.exists(Path.fromString(USERAPP_ZK_SUBPATH).append(HOSTS)))
return getUserAppData(HOSTS);
return null;
}
@@ -133,9 +142,9 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public List<NamedReader> getSchemas() {
List<NamedReader> schemas = new ArrayList<>();
- for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
+ for (String sd : zkApplication.getChildren(Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR))) {
if (sd.endsWith(SD_NAME_SUFFIX))
- schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
+ schemas.add(zkApplication.getNamedReader(sd, Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR).append(sd)));
}
return schemas;
}
@@ -161,7 +170,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
private Reader retrieveConfigDefReader(String def) {
try {
- return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def);
+ return zkApplication.getNamedReader("configdefinition", Path.fromString(DEFCONFIGS_ZK_SUBPATH).append(def));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Could not retrieve config definition " + def, e);
}
@@ -171,7 +180,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() {
Map<ConfigDefinitionKey, UnparsedConfigDefinition> ret = new LinkedHashMap<>();
- List<String> allDefs = zkApplication.getChildren(ConfigCurator.DEFCONFIGS_ZK_SUBPATH);
+ List<String> allDefs = zkApplication.getChildren(Path.fromString(DEFCONFIGS_ZK_SUBPATH));
for (String nodeName : allDefs) {
ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName);
@@ -201,7 +210,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
*/
@Override
public List<NamedReader> getFiles(Path relativePath, String suffix, boolean recurse) {
- return zkApplication.getAllDataFromDirectory(ConfigCurator.USERAPP_ZK_SUBPATH + '/' + relativePath.getRelative(), suffix, recurse);
+ return zkApplication.getAllDataFromDirectory(Path.fromString(USERAPP_ZK_SUBPATH).append(relativePath), suffix, recurse);
}
@Override
@@ -226,7 +235,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES.getName()); }
private Optional<Reader> optionalFile(String file) {
- if (zkApplication.exists(ConfigCurator.USERAPP_ZK_SUBPATH, file))
+ if (zkApplication.exists(Path.fromString(USERAPP_ZK_SUBPATH).append(file)))
return Optional.of(getUserAppData(file));
else
return Optional.empty();
@@ -246,17 +255,17 @@ public class ZKApplicationPackage implements ApplicationPackage {
}
private Reader getUserAppData(String node) {
- return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH, node);
+ return zkApplication.getDataReader(Path.fromString(USERAPP_ZK_SUBPATH).append(node));
}
@Override
public Reader getRankingExpression(String name) {
- return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name);
+ return zkApplication.getDataReader(Path.fromString(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR).append(name));
}
@Override
public File getFileReference(Path pathRelativeToAppDir) {
- String path = ConfigCurator.USERAPP_ZK_SUBPATH + "/" + pathRelativeToAppDir.getRelative();
+ Path path = Path.fromString(USERAPP_ZK_SUBPATH).append(pathRelativeToAppDir);
// File does not exist: Manufacture a non-existing file
if ( ! zkApplication.exists(path)) return new File(pathRelativeToAppDir.getRelative());
@@ -266,8 +275,8 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public void validateIncludeDir(String dirName) {
- String fullPath = ConfigCurator.USERAPP_ZK_SUBPATH + "/" + dirName;
- if ( ! zkApplication.exists(fullPath)) {
+ Path path = Path.fromString(USERAPP_ZK_SUBPATH).append(dirName);
+ if ( ! zkApplication.exists(path)) {
throw new IllegalArgumentException("Cannot include directory '" + dirName +
"', as it does not exist in ZooKeeper!");
}
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index d0e366b11a1..13a660221fd 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -30,7 +30,6 @@
<component id="com.yahoo.vespa.config.server.host.HostRegistry" bundle="configserver" />
<component id="com.yahoo.vespa.config.server.ApplicationRepository" bundle="configserver" />
<component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" />
- <component id="com.yahoo.vespa.config.server.zookeeper.ConfigCurator" bundle="configserver" />
<component id="com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker" bundle="configserver" />
<component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" />
<component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" />
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 934440f03d1..1b31d02d222 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -21,6 +21,7 @@ import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.io.IOUtils;
import com.yahoo.jdisc.Metric;
+import com.yahoo.path.Path;
import com.yahoo.test.ManualClock;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigKey;
@@ -43,7 +44,6 @@ import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore;
import com.yahoo.vespa.config.server.tenant.Tenant;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.util.ConfigUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
@@ -104,7 +104,6 @@ public class ApplicationRepositoryTest {
private OrchestratorMock orchestrator;
private TimeoutBudget timeoutBudget;
private Curator curator;
- private ConfigCurator configCurator;
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -115,7 +114,6 @@ public class ApplicationRepositoryTest {
@Before
public void setup() throws IOException {
curator = new MockCurator();
- configCurator = ConfigCurator.create(curator);
ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder()
.payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED)
.configServerDBDir(temporaryFolder.newFolder().getAbsolutePath())
@@ -331,8 +329,8 @@ public class ApplicationRepositoryTest {
assertNotNull(applicationData.getApplicationId());
assertNotNull(sessionRepository.getLocalSession(sessionId));
assertNotNull(applicationRepository.getActiveSession(applicationId()));
- String sessionNode = sessionRepository.getSessionPath(sessionId).getAbsolute();
- assertTrue(configCurator.exists(sessionNode));
+ Path sessionNode = sessionRepository.getSessionPath(sessionId);
+ assertTrue(curator.exists(sessionNode));
TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs();
File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId));
assertTrue(sessionFile.exists());
@@ -344,8 +342,8 @@ public class ApplicationRepositoryTest {
assertTrue(provisioner.removed());
assertEquals(tenant.getName(), provisioner.lastApplicationId().tenant());
assertEquals(applicationId(), provisioner.lastApplicationId());
- assertTrue(configCurator.exists(sessionNode));
- assertEquals(Session.Status.DELETE.name(), configCurator.getData(sessionNode + "/sessionState"));
+ assertTrue(curator.exists(sessionNode));
+ assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get()));
assertTrue(sessionFile.exists());
assertFalse(applicationRepository.delete(applicationId()));
@@ -395,8 +393,8 @@ public class ApplicationRepositoryTest {
assertTrue(applicationRepository.delete(applicationId()));
// Session should be in state DELETE
- String sessionNode = sessionRepository.getSessionPath(sessionId).getAbsolute();
- assertEquals(Session.Status.DELETE.name(), configCurator.getData(sessionNode + "/sessionState"));
+ Path sessionNode = sessionRepository.getSessionPath(sessionId);
+ assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get()));
assertNotNull(sessionRepository.getRemoteSession(sessionId)); // session still exists
assertNull(applicationRepository.getActiveSession(applicationId())); // but it is not active
try {
@@ -472,7 +470,6 @@ public class ApplicationRepositoryTest {
sessionId,
FilesApplicationPackage.fromFile(testApp),
new SessionZooKeeperClient(curator,
- configCurator,
tenant1,
sessionId,
ConfigUtils.getCanonicalHostName()));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
index e20363af4e9..2b1fa9acd93 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
@@ -14,8 +14,9 @@ import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.path.Path;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
+import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Before;
import org.junit.Rule;
@@ -31,6 +32,10 @@ import java.util.Map;
import java.util.Optional;
import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.DEFCONFIGS_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -46,13 +51,13 @@ public class ZooKeeperClientTest {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private ConfigCurator zk;
- private final String appPath = "/1";
+ private Curator zk;
+ private final Path appPath = Path.fromString("/1");
@Before
public void setupZK() throws IOException {
- zk = ConfigCurator.create(new MockCurator());
- ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), Path.fromString(appPath));
+ zk = new MockCurator();
+ ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), appPath);
ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"),
new DeployData("foo",
"/bar/baz",
@@ -81,30 +86,30 @@ public class ZooKeeperClientTest {
@Test
public void testInitZooKeeper() {
- ConfigCurator zk = ConfigCurator.create(new MockCurator());
+ Curator zk = new MockCurator();
BaseDeployLogger logger = new BaseDeployLogger();
long generation = 1L;
ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, Path.fromString("/1"));
zooKeeperClient.initialize();
- String appPath = "/";
+ Path appPath = Path.fromString("/");
assertThat(zk.getChildren(appPath).size(), is(1));
- assertTrue(zk.exists("/" + generation));
- String currentAppPath = appPath + generation;
- assertTrue(zk.exists(currentAppPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")));
+ Path currentAppPath = appPath.append(String.valueOf(generation));
+ assertTrue(zk.exists(currentAppPath));
+ assertTrue(zk.exists(currentAppPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
assertThat(zk.getChildren(currentAppPath).size(), is(4));
}
@Test
public void testFeedDefFilesToZooKeeper() {
- String defsPath = appPath + ConfigCurator.DEFCONFIGS_ZK_SUBPATH;
- assertTrue(zk.exists(appPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")));
+ Path defsPath = appPath.append(DEFCONFIGS_ZK_SUBPATH);
+ assertTrue(zk.exists(appPath.append(DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
List<String> children = zk.getChildren(defsPath);
assertEquals(defsPath + " children", 1, children.size());
Collections.sort(children);
assertThat(children.get(0), is("a.b.test2"));
- assertTrue(zk.exists(appPath, ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", "")));
- String userDefsPath = appPath + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH;
+ assertTrue(zk.exists(appPath.append(USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))));
+ Path userDefsPath = appPath.append(USER_DEFCONFIGS_ZK_SUBPATH);
children = zk.getChildren(userDefsPath);
assertThat(children.size(), is(1));
Collections.sort(children);
@@ -113,8 +118,9 @@ public class ZooKeeperClientTest {
@Test
public void testFeedAppMetaDataToZooKeeper() {
- assertTrue(zk.exists(appPath, ConfigCurator.META_ZK_PATH));
- ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(zk.getData(appPath, ConfigCurator.META_ZK_PATH));
+ assertTrue(zk.exists(appPath.append(META_ZK_PATH)));
+ ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(
+ Utf8.toString(zk.getData(appPath.append(META_ZK_PATH)).get()));
assertTrue(metaData.getChecksum().length() > 0);
assertTrue(metaData.isInternalRedeploy());
assertThat(metaData.getDeployedByUser(), is("foo"));
@@ -126,34 +132,34 @@ public class ZooKeeperClientTest {
@Test
public void testVersionedFileRegistry() {
- String fileRegPath = appPath + "/" + ZKApplicationPackage.fileRegistryNode;
+ Path fileRegPath = appPath.append(ZKApplicationPackage.fileRegistryNode);
assertTrue(zk.exists(fileRegPath));
- assertTrue(zk.exists(fileRegPath + "/1.2.3"));
- assertTrue(zk.exists(fileRegPath + "/3.2.1"));
+ assertTrue(zk.exists(fileRegPath.append("/1.2.3")));
+ assertTrue(zk.exists(fileRegPath.append("/3.2.1")));
// assertNull("Data at " + fileRegPath, zk.getData(fileRegPath)); Not null any more .. hm
}
@Test
public void include_dirs_are_written_to_ZK() {
- assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/" + "dir1", "default.xml"));
- assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain2.xml"));
- assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain3.xml"));
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("dir1").append("default.xml")));
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain2.xml")));
+ assertTrue(zk.exists(appPath.append(USERAPP_ZK_SUBPATH).append("nested").append("dir2").append("chain3.xml")));
}
@Test
public void search_chain_dir_written_to_ZK() {
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml").getAbsolute()));
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml").getAbsolute()));
- assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml").getAbsolute()));
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml")));
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml")));
+ assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml")));
}
private Path appPath() {
- return Path.fromString(appPath).append(ConfigCurator.USERAPP_ZK_SUBPATH);
+ return appPath.append(USERAPP_ZK_SUBPATH);
}
@Test
public void testWritingHostNamesToZooKeeper() throws IOException {
- ConfigCurator zk = ConfigCurator.create(new MockCurator());
+ Curator zk = new MockCurator();
BaseDeployLogger logger = new BaseDeployLogger();
Path app = Path.fromString("/1");
ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app);
@@ -163,9 +169,9 @@ public class ZooKeeperClientTest {
ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2);
zooKeeperClient.write(AllocatedHosts.withHosts(hosts));
Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode);
- assertTrue(zk.exists(hostsPath.getAbsolute()));
+ assertTrue(zk.exists(hostsPath));
- AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute()));
+ AllocatedHosts deserialized = fromJson(zk.getData(hostsPath).get());
assertEquals(hosts, deserialized.getHosts());
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
index 641fbe5bf41..ebbb10c2d2a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java
@@ -1,15 +1,16 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.deploy;
+import com.yahoo.component.Version;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.model.application.provider.*;
+import com.yahoo.config.model.application.provider.FilesApplicationPackage;
+import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.component.Version;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -33,7 +34,7 @@ public class ZooKeeperDeployerTest {
@Test
public void require_that_deployer_is_initialized() throws IOException {
- ConfigCurator zkfacade = ConfigCurator.create(new MockCurator());
+ Curator curator = new MockCurator();
File serverdbDir = folder.newFolder("serverdb");
File defsDir = new File(serverdbDir, "serverdefs");
try {
@@ -42,21 +43,22 @@ public class ZooKeeperDeployerTest {
e.printStackTrace();
fail();
}
- deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/1"));
- deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/2"));
+ deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/1"));
+ deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), curator, Path.fromString("/2"));
}
- public void deploy(ApplicationPackage applicationPackage, ConfigCurator configCurator, Path appPath) throws IOException {
+ public void deploy(ApplicationPackage applicationPackage, Curator curator, Path appPath) throws IOException {
MockDeployLogger logger = new MockDeployLogger();
- ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, appPath);
+ ZooKeeperClient client = new ZooKeeperClient(curator, logger, appPath);
ZooKeeperDeployer deployer = new ZooKeeperDeployer(client);
deployer.deploy(applicationPackage, Collections.singletonMap(new Version(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet()));
- assertTrue(configCurator.exists(appPath.getAbsolute()));
+ assertTrue(curator.exists(appPath));
}
private static class MockDeployLogger implements DeployLogger {
@Override
public void log(Level level, String message) { }
}
+
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java
deleted file mode 100644
index 0451ef84e09..00000000000
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.session;
-
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.model.test.MockApplicationPackage;
-import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
-import com.yahoo.vespa.config.util.ConfigUtils;
-import com.yahoo.vespa.curator.Curator;
-
-import java.util.Optional;
-
-/**
- * Overrides application package fetching, because this part is hard to do without feeding a full app.
- *
- * @author Ulf Lilleengen
- */
-public class MockSessionZKClient extends SessionZooKeeperClient {
-
- private final ApplicationPackage app;
- private Optional<AllocatedHosts> info = Optional.empty();
-
- public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId) {
- this(curator, tenantName, sessionId, (ApplicationPackage) null);
- }
-
- public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, Optional<AllocatedHosts> allocatedHosts) {
- this(curator, tenantName, sessionId);
- this.info = allocatedHosts;
- }
-
- MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, ApplicationPackage application) {
- super(curator,
- ConfigCurator.create(curator),
- tenantName,
- sessionId,
- ConfigUtils.getCanonicalHostName());
- this.app = application;
- curator.create(TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)));
- }
-
- @Override
- public ApplicationPackage loadApplicationPackage() {
- if (app != null) return app;
- return new MockApplicationPackage.Builder().withEmptyServices().build();
- }
-
- @Override
- AllocatedHosts getAllocatedHosts() {
- return info.orElseThrow(() -> new IllegalStateException("Could not find allocated hosts"));
- }
-
-}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 4995dc0decc..a9e7ffd5b5a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -39,7 +39,7 @@ import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
+import com.yahoo.vespa.config.server.zookeeper.ZKApplication;
import com.yahoo.vespa.config.util.ConfigUtils;
import com.yahoo.vespa.curator.mock.MockCurator;
import com.yahoo.vespa.flags.InMemoryFlagSource;
@@ -87,7 +87,6 @@ public class SessionPreparerTest {
Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build();
private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
private MockCurator curator;
- private ConfigCurator configCurator;
private SessionPreparer preparer;
private final MockSecretStore secretStore = new MockSecretStore();
private ConfigserverConfig configserverConfig;
@@ -101,7 +100,6 @@ public class SessionPreparerTest {
@Before
public void setUp() throws IOException {
curator = new MockCurator();
- configCurator = ConfigCurator.create(curator);
configserverConfig = new ConfigserverConfig.Builder()
.fileReferencesDir(folder.newFolder().getAbsolutePath())
.configServerDBDir(folder.newFolder().getAbsolutePath())
@@ -162,7 +160,7 @@ public class SessionPreparerTest {
.build(),
sessionId);
Path sessionPath = sessionPath(sessionId);
- assertFalse(configCurator.exists(sessionPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute()));
+ assertFalse(curator.exists(sessionPath.append(ZKApplication.USERAPP_ZK_SUBPATH).append("services.xml")));
}
@Test
@@ -181,7 +179,7 @@ public class SessionPreparerTest {
@Test
public void require_that_application_is_prepared() throws Exception {
prepare(testApp);
- assertTrue(configCurator.exists(sessionPath(1).append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute()));
+ assertTrue(curator.exists(sessionPath(1).append(ZKApplication.USERAPP_ZK_SUBPATH).append("services.xml")));
}
@Test(expected = InvalidApplicationException.class)
@@ -234,7 +232,7 @@ public class SessionPreparerTest {
@Test
public void require_that_file_reference_of_application_package_is_written_to_zk() throws Exception {
prepare(testApp);
- assertTrue(configCurator.exists(sessionPath(1).append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute()));
+ assertTrue(curator.exists(sessionPath(1).append(APPLICATION_PACKAGE_REFERENCE_PATH)));
}
@Test
@@ -376,7 +374,7 @@ public class SessionPreparerTest {
}
private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) {
- return new SessionZooKeeperClient(curator, configCurator, applicationId().tenant(), sessionId, ConfigUtils.getCanonicalHostName());
+ return new SessionZooKeeperClient(curator, applicationId().tenant(), sessionId, ConfigUtils.getCanonicalHostName());
}
private Path sessionPath(long sessionId) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java
index 3fa00e3ecc3..2c131e56d67 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java
@@ -28,7 +28,6 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException;
import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.util.ConfigUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
@@ -285,7 +284,6 @@ public class SessionRepositoryTest {
private void createSession(long sessionId, boolean wait) {
SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator,
- ConfigCurator.create(curator),
tenantName,
sessionId,
ConfigUtils.getCanonicalHostName());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java
index f977011efbc..57ef55cc890 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.session;
import com.yahoo.config.FileReference;
@@ -9,7 +9,6 @@ import com.yahoo.config.provision.TenantName;
import com.yahoo.path.Path;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.util.ConfigUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
@@ -22,6 +21,7 @@ import java.time.Instant;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.SESSIONSTATE_ZK_SUBPATH;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -35,7 +35,6 @@ public class SessionZooKeeperClientTest {
private static final TenantName tenantName = TenantName.defaultName();
private Curator curator;
- private ConfigCurator configCurator;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@@ -43,7 +42,6 @@ public class SessionZooKeeperClientTest {
@Before
public void setup() {
curator = new MockCurator();
- configCurator = ConfigCurator.create(curator);
curator.create(sessionsPath());
}
@@ -68,16 +66,16 @@ public class SessionZooKeeperClientTest {
int sessionId = 2;
SessionZooKeeperClient zkc = createSessionZKClient(sessionId);
zkc.writeStatus(Session.Status.NEW);
- String path = sessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute();
- assertTrue(configCurator.exists(path));
- assertThat(configCurator.getData(path), is("NEW"));
+ Path path = sessionPath(sessionId).append(SESSIONSTATE_ZK_SUBPATH);
+ assertTrue(curator.exists(path));
+ assertThat(Utf8.toString(curator.getData(path).get()), is("NEW"));
}
@Test
public void require_that_status_is_read_from_zk() {
int sessionId = 3;
SessionZooKeeperClient zkc = createSessionZKClient(sessionId);
- curator.set(sessionPath(sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH), Utf8.toBytes("PREPARE"));
+ curator.set(sessionPath(sessionId).append(SESSIONSTATE_ZK_SUBPATH), Utf8.toBytes("PREPARE"));
assertThat(zkc.readStatus(), is(Session.Status.PREPARE));
}
@@ -91,9 +89,9 @@ public class SessionZooKeeperClientTest {
int sessionId = 3;
SessionZooKeeperClient zkc = createSessionZKClient(sessionId);
zkc.writeApplicationId(id);
- String path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute();
- assertTrue(configCurator.exists(path));
- assertThat(configCurator.getData(path), is(id.serializedForm()));
+ Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH);
+ assertTrue(curator.exists(path));
+ assertThat(Utf8.toString(curator.getData(path).get()), is(id.serializedForm()));
}
@Test
@@ -163,15 +161,14 @@ public class SessionZooKeeperClientTest {
private void assertApplicationIdParse(long sessionId, String idString, String expectedIdString) {
SessionZooKeeperClient zkc = createSessionZKClient(sessionId);
- String path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute();
- configCurator.putData(path, idString);
+ Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH);
+ curator.set(path, Utf8.toBytes(idString));
ApplicationId applicationId = zkc.readApplicationId().get();
assertThat(applicationId.serializedForm(), is(expectedIdString));
}
private SessionZooKeeperClient createSessionZKClient(long sessionId) {
SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator,
- ConfigCurator.create(curator),
tenantName,
sessionId,
ConfigUtils.getCanonicalHostName());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
index 464b3d1ab64..4cf188a0b33 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.server.tenant;
import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.component.Version;
import com.yahoo.concurrent.InThreadExecutorService;
import com.yahoo.concurrent.StripedExecutor;
@@ -27,7 +28,6 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import com.yahoo.vespa.flags.InMemoryFlagSource;
@@ -208,7 +208,7 @@ public class TenantRepositoryTest {
public FailingDuringBootstrapTenantRepository(ConfigserverConfig configserverConfig) {
super(new HostRegistry(),
- ConfigCurator.create(new MockCurator()),
+ new MockCurator(),
Metrics.createTestMetrics(),
new StripedExecutor<>(new InThreadExecutorService()),
new StripedExecutor<>(new InThreadExecutorService()),
@@ -224,7 +224,8 @@ public class TenantRepositoryTest {
new ModelFactoryRegistry(List.of(new VespaModelFactory(new NullConfigModelRegistry()))),
new TestConfigDefinitionRepo(),
new TenantApplicationsTest.MockReloadListener(),
- new MockTenantListener());
+ new MockTenantListener(),
+ new ZookeeperServerConfig.Builder().myid(0).build());
}
@Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java
index 687d58fd23b..47e71f6d0fa 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.server.tenant;
import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.concurrent.InThreadExecutorService;
import com.yahoo.concurrent.StripedExecutor;
import com.yahoo.config.model.NullConfigModelRegistry;
@@ -17,7 +18,6 @@ import com.yahoo.vespa.config.server.host.HostRegistry;
import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import com.yahoo.vespa.flags.FlagSource;
@@ -47,7 +47,7 @@ public class TestTenantRepository extends TenantRepository {
ReloadListener reloadListener,
TenantListener tenantListener) {
super(hostRegistry,
- ConfigCurator.create(curator),
+ curator,
metrics,
new StripedExecutor<>(new InThreadExecutorService()),
new StripedExecutor<>(new InThreadExecutorService()),
@@ -63,7 +63,8 @@ public class TestTenantRepository extends TenantRepository {
modelFactoryRegistry,
configDefinitionRepo,
reloadListener,
- tenantListener);
+ tenantListener,
+ new ZookeeperServerConfig.Builder().myid(0).build());
}
public static class Builder {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java
deleted file mode 100644
index 9a8aec72564..00000000000
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java
+++ /dev/null
@@ -1,243 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.zookeeper;
-
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.curator.mock.MockCurator;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import static org.junit.Assert.*;
-
-/**
- * Tests the ZKFacade using a curator mock.
- *
- * @author hmusum
- */
-public class ConfigCuratorTest {
-
- private final String defKey1 = "attributes";
-
- private final String payload1 = "attribute[5]\n" +
- "attribute[0].name Popularity\n" +
- "attribute[0].datatype string\n" +
- "attribute[0].collectiontype single\n" +
- "attribute[0].removeifzero false\n" +
- "attribute[0].createifnonexistent false\n" +
- "attribute[0].loadtype \"always\"\n" +
- "attribute[0].uniqueonly false\n" +
- "attribute[0].sparse false\n" +
- "attribute[0].noupdate false\n" +
- "attribute[0].fastsearch false\n" +
- "attribute[0].fastaggregate false\n" +
- "attribute[0].fastersearch false\n" +
- "attribute[1].name atA\n" +
- "attribute[1].datatype string\n" +
- "attribute[1].collectiontype weightedset\n" +
- "attribute[1].removeifzero false\n" +
- "attribute[1].createifnonexistent false\n" +
- "attribute[1].loadtype \"always\"\n" +
- "attribute[1].uniqueonly false\n" +
- "attribute[1].sparse false\n" +
- "attribute[1].noupdate false\n" +
- "attribute[1].fastsearch true\n" +
- "attribute[1].fastaggregate false\n" +
- "attribute[1].fastersearch false\n" +
- "attribute[2].name default_fieldlength\n" +
- "attribute[2].datatype uint32\n" +
- "attribute[2].collectiontype single\n" +
- "attribute[2].removeifzero false\n" +
- "attribute[2].createifnonexistent false\n" +
- "attribute[2].loadtype \"always\"\n" +
- "attribute[2].uniqueonly false\n" +
- "attribute[2].sparse false\n" +
- "attribute[2].noupdate true\n" +
- "attribute[2].fastsearch false\n" +
- "attribute[2].fastaggregate false\n" +
- "attribute[2].fastersearch false\n" +
- "attribute[3].name default_literal_fieldlength\n" +
- "attribute[3].datatype uint32\n" +
- "attribute[3].collectiontype single\n" +
- "attribute[3].removeifzero false\n" +
- "attribute[3].createifnonexistent false\n" +
- "attribute[3].loadtype \"always\"\n" +
- "attribute[3].uniqueonly false\n" +
- "attribute[3].sparse false\n" +
- "attribute[3].noupdate true\n" +
- "attribute[3].fastsearch false\n" +
- "attribute[3].fastaggregate false\n" +
- "attribute[3].fastersearch false\n" +
- "attribute[4].name artist_fieldlength\n" +
- "attribute[4].datatype uint32\n" +
- "attribute[4].collectiontype single\n" +
- "attribute[4].removeifzero false\n" +
- "attribute[4].createifnonexistent false\n" +
- "attribute[4].loadtype \"always\"\n" +
- "attribute[4].uniqueonly false\n" +
- "attribute[4].sparse false\n" +
- "attribute[4].noupdate true\n" +
- "attribute[4].fastsearch false\n" +
- "attribute[4].fastaggregate false\n" +
- "attribute[4].fastersearch false\n";
-
- private final String payload3 = "attribute[5]\n" +
- "attribute[0].name Popularity\n" +
- "attribute[0].datatype String\n" +
- "attribute[0].collectiontype single\n" +
- "attribute[0].removeifzero false\n" +
- "attribute[0].createifnonexistent false\n" +
- "attribute[0].loadtype \"always\"\n" +
- "attribute[0].uniqueonly false\n" +
- "attribute[0].sparse false\n" +
- "attribute[0].noupdate false\n" +
- "attribute[0].fastsearch false\n" +
- "attribute[0].fastaggregate false\n" +
- "attribute[0].fastersearch false\n" +
- "attribute[1].name atA\n" +
- "attribute[1].datatype string\n" +
- "attribute[1].collectiontype weightedset\n" +
- "attribute[1].removeifzero false\n" +
- "attribute[1].createifnonexistent false\n" +
- "attribute[1].loadtype \"always\"\n" +
- "attribute[1].uniqueonly false\n" +
- "attribute[1].sparse false\n" +
- "attribute[1].noupdate false\n" +
- "attribute[1].fastsearch true\n" +
- "attribute[1].fastaggregate false\n" +
- "attribute[1].fastersearch false\n" +
- "attribute[2].name default_fieldlength\n" +
- "attribute[2].datatype uint32\n" +
- "attribute[2].collectiontype single\n" +
- "attribute[2].removeifzero false\n" +
- "attribute[2].createifnonexistent false\n" +
- "attribute[2].loadtype \"always\"\n" +
- "attribute[2].uniqueonly false\n" +
- "attribute[2].sparse false\n" +
- "attribute[2].noupdate true\n" +
- "attribute[2].fastsearch false\n" +
- "attribute[2].fastaggregate false\n" +
- "attribute[2].fastersearch false\n" +
- "attribute[3].name default_literal_fieldlength\n" +
- "attribute[3].datatype uint32\n" +
- "attribute[3].collectiontype single\n" +
- "attribute[3].removeifzero false\n" +
- "attribute[3].createifnonexistent false\n" +
- "attribute[3].loadtype \"always\"\n" +
- "attribute[3].uniqueonly false\n" +
- "attribute[3].sparse false\n" +
- "attribute[3].noupdate true\n" +
- "attribute[3].fastsearch false\n" +
- "attribute[3].fastaggregate false\n" +
- "attribute[3].fastersearch false\n" +
- "attribute[4].name artist_fieldlength\n" +
- "attribute[4].datatype uint32\n" +
- "attribute[4].collectiontype single\n" +
- "attribute[4].removeifzero false\n" +
- "attribute[4].createifnonexistent false\n" +
- "attribute[4].loadtype \"always\"\n" +
- "attribute[4].uniqueonly false\n" +
- "attribute[4].sparse false\n" +
- "attribute[4].noupdate true\n" +
- "attribute[4].fastsearch false\n" +
- "attribute[4].fastaggregate false\n" +
- "attribute[4].fastersearch false\n";
-
- private void initAndClearZK(ConfigCurator zkIf) {
- zkIf.initAndClear(ConfigCurator.DEFCONFIGS_ZK_SUBPATH);
- zkIf.initAndClear(ConfigCurator.USERAPP_ZK_SUBPATH);
- }
-
- private ConfigCurator deployApp() {
- ConfigCurator zkIf = create();
- initAndClearZK(zkIf);
- zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1, payload1);
- // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, cfgKey1, payload3);
- String partitionsDef = "version=7\\n" +
- "dataset[].id int\\n" +
- "dataset[].partbits int default=6";
- zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, "partitions", partitionsDef);
- String partitionsUser = "dataset[0].partbits 8\\n";
- // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, "partitions", partitionsUser);
- return zkIf;
- }
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @Test
- public void testZKInterface() {
- ConfigCurator zkIf = create();
- zkIf.putData("", "test", "foo");
- zkIf.putData("/test", "me", "bar");
- zkIf.putData("", "test;me;now,then", "baz");
- assertEquals(zkIf.getData("", "test"), "foo");
- assertEquals(zkIf.getData("/test", "me"), "bar");
- assertEquals(zkIf.getData("", "test;me;now,then"), "baz");
- }
-
- @Test
- public void testNonExistingPath() {
- ConfigCurator configCurator = create();
-
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Cannot read data from path /non-existing, it does not exist");
- configCurator.getData("/non-existing");
- }
-
- @Test
- public void testWatcher() {
- ConfigCurator zkIf = create();
-
- zkIf.putData("", "test", "foo");
- String data = zkIf.getData("/test");
- assertEquals(data, "foo");
- zkIf.putData("", "/test", "bar");
- data = zkIf.getData("/test");
- assertEquals(data, "bar");
-
- zkIf.getChildren("/");
- zkIf.putData("", "test2", "foo2");
- }
-
- private ConfigCurator create() {
- return ConfigCurator.create(new MockCurator());
- }
-
- @Test
- public void testGetDeployedData() {
- ConfigCurator zkIf = deployApp();
- assertEquals(zkIf.getData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1), payload1);
- }
-
- @Test
- public void testEmptyData() {
- ConfigCurator zkIf = create();
- zkIf.createNode("/empty", "data");
- assertEquals("", zkIf.getData("/empty", "data"));
- }
-
- @Test
- public void testRecursiveDelete() {
- ConfigCurator configCurator = create();
- configCurator.putData("/foo", Utf8.toBytes("sadsdfsdfsdfsdf"));
- configCurator.putData("/foo/bar", Utf8.toBytes("dsfsdffds"));
- configCurator.putData("/foo/baz",
- Utf8.toBytes("sdf\u00F8l ksdfl skdflsk dflsdkfd welkr3k lkr e4kt4 54l4l353k l534klk3lk4l33k5l 353l4k l43k l4k"));
- configCurator.putData("/foo/bar/dill", Utf8.toBytes("sdfsfe 23 42 3 3 2342"));
- configCurator.putData("/foo", Utf8.toBytes("sdcfsdfsdf"));
- configCurator.putData("/foo", Utf8.toBytes("sdcfsd sdfdffsdf"));
- configCurator.deleteRecurse("/foo");
- assertFalse(configCurator.exists("/foo"));
- assertFalse(configCurator.exists("/foo/bar"));
- assertFalse(configCurator.exists("/foo/bar/dill"));
- assertFalse(configCurator.exists("/foo/bar/baz"));
- try {
- configCurator.getChildren("/foo");
- fail("Got children from nonexisting ZK path");
- } catch (RuntimeException e) {
- assertTrue(e.getCause().getMessage().matches(".*NoNode.*"));
- }
- configCurator.deleteRecurse("/nonexisting");
- }
-
-}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java
index f745e023126..656e81a4c2f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java
@@ -1,6 +1,8 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.zookeeper;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Test;
@@ -14,12 +16,12 @@ public class InitializedCounterTest {
@Test
public void requireThatCounterIsInitializedFromNumberOfSessions() {
- ConfigCurator configCurator = ConfigCurator.create(new MockCurator());
- configCurator.createNode("/sessions");
- configCurator.createNode("/sessions/1");
- configCurator.createNode("/sessions/2");
+ Curator curator = new MockCurator();
+ curator.create(Path.fromString("/sessions"));
+ curator.create(Path.fromString("/sessions/1"));
+ curator.create(Path.fromString("/sessions/2"));
- InitializedCounter counter = new InitializedCounter(configCurator, "/counter", "/sessions");
+ InitializedCounter counter = new InitializedCounter(curator, Path.fromString("/counter"), Path.fromString("/sessions"));
assertThat(counter.counter.get(), is(2L));
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java
index a34c17dc909..b149aa94441 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java
@@ -1,9 +1,11 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.zookeeper;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.application.api.ApplicationFileTest;
import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
@@ -11,6 +13,7 @@ import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
import static org.junit.Assert.assertTrue;
/**
@@ -21,20 +24,20 @@ public class ZKApplicationFileTest extends ApplicationFileTest {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private void feed(ConfigCurator zk, File dirToFeed) {
+ private void feed(Curator curator, File dirToFeed) {
assertTrue(dirToFeed.isDirectory());
- String appPath = "/0";
- ZKApplicationPackageTest.feedZooKeeper(zk, dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true);
- zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles");
+ Path appPath = Path.fromString("/0");
+ ZKApplicationPackageTest.feedZooKeeper(curator, dirToFeed, appPath.append(USERAPP_ZK_SUBPATH), null, true);
+ curator.set(appPath.append(ZKApplicationPackage.fileRegistryNode), Utf8.toBytes("dummyfiles"));
}
@Override
public ApplicationFile getApplicationFile(Path path) throws IOException{
- ConfigCurator configCurator = ConfigCurator.create(new MockCurator());
+ Curator curator = new MockCurator();
File tmp = temporaryFolder.newFolder();
writeAppTo(tmp);
- feed(configCurator, tmp);
- return new ZKApplicationFile(path, new ZKApplication(configCurator, Path.fromString("/0")));
+ feed(curator, tmp);
+ return new ZKApplicationFile(path, new ZKApplication(curator, Path.fromString("/0")));
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
index 458cdb82066..4b4b4b92627 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
@@ -14,6 +14,7 @@ import com.yahoo.config.provisioning.FlavorsConfig;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
import com.yahoo.text.Utf8;
+import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Before;
import org.junit.Rule;
@@ -31,6 +32,8 @@ import java.util.Set;
import java.util.regex.Pattern;
import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.META_ZK_PATH;
+import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -59,20 +62,20 @@ public class ZKApplicationPackageTest {
Optional.of(DockerImage.fromString(dockerImage)))));
}
- private ConfigCurator configCurator;
+ private Curator curator;
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
@Before
public void setup() {
- configCurator = ConfigCurator.create(new MockCurator());
+ curator = new MockCurator();
}
@Test
public void testBasicZKFeed() throws IOException {
- feed(configCurator, new File(APP));
- ZKApplicationPackage zkApp = new ZKApplicationPackage(configCurator, Path.fromString("/0"));
+ feed(curator, new File(APP));
+ ZKApplicationPackage zkApp = new ZKApplicationPackage(curator, Path.fromString("/0"));
assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches());
assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches());
assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches());
@@ -103,13 +106,14 @@ public class ZKApplicationPackageTest {
assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get());
}
- private void feed(ConfigCurator zk, File dirToFeed) throws IOException {
+ private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException {
assertTrue(dirToFeed.isDirectory());
- feedZooKeeper(zk, dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true);
+ Path sessionPath = Path.fromString("/0");
+ feedZooKeeper(zk, dirToFeed, sessionPath.append(USERAPP_ZK_SUBPATH), null, true);
String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}";
- zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData);
- zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles");
- zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS));
+ zk.set(sessionPath.append(META_ZK_PATH), Utf8.toBytes(metaData));
+ zk.set(sessionPath.append(ZKApplicationPackage.fileRegistryNode).append("/3.0.0"), Utf8.toBytes("dummyfiles"));
+ zk.set(sessionPath.append(ZKApplicationPackage.allocatedHostsNode), toJson(ALLOCATED_HOSTS));
}
private static class MockNodeFlavors extends NodeFlavors{
@@ -131,7 +135,7 @@ public class ZKApplicationPackageTest {
* @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper
* @param recurse recurse subdirectories
*/
- static void feedZooKeeper(ConfigCurator zk, File dir, String path, FilenameFilter filenameFilter, boolean recurse) {
+ static void feedZooKeeper(com.yahoo.vespa.curator.Curator zk, File dir, Path path, FilenameFilter filenameFilter, boolean recurse) {
try {
if (filenameFilter == null) {
filenameFilter = acceptsAllFileNameFilter;
@@ -141,12 +145,13 @@ public class ZKApplicationPackageTest {
}
for (File file : listFiles(dir, filenameFilter)) {
if (file.getName().startsWith(".")) continue; //.svn , .git ...
+ Path filePath = path.append(file.getName());
if (file.isFile()) {
- String contents = IOUtils.readFile(file);
- zk.putData(path, file.getName(), contents);
+ byte[] contents = IOUtils.readFileBytes(file);
+ zk.set(filePath, contents);
} else if (recurse && file.isDirectory()) {
- zk.createNode(path, file.getName());
- feedZooKeeper(zk, file, path + '/' + file.getName(), filenameFilter, recurse);
+ zk.create(filePath);
+ feedZooKeeper(zk, file, filePath, filenameFilter, recurse);
}
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
index fa5a72eea52..9beee666a74 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
@@ -54,7 +54,7 @@ public class CuratorDatabase {
public CuratorDatabase(Curator curator, Path root, boolean useCache) {
this.useCache = useCache;
this.curator = curator;
- changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter").getAbsolute());
+ changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter"));
cache.set(newCache(changeGenerationCounter.get()));
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index c37db2fa526..da4ab528630 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -82,7 +82,7 @@ public class CuratorDatabaseClient {
this.nodeSerializer = new NodeSerializer(flavors, nodeCacheSize);
this.db = new CuratorDatabase(curator, root, useCache);
this.clock = clock;
- this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute());
+ this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter"));
initZK();
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
index 7ee65ebcd0b..467ee90176d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
@@ -1,6 +1,7 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.orchestrator.status;
+import com.yahoo.path.Path;
import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.curator.Curator;
@@ -15,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong;
* @author hakonhall
*/
public class HostInfosCache implements HostInfosService {
- final static String HOST_STATUS_CACHE_COUNTER_PATH = "/vespa/host-status-service-cache-counter";
+ final static Path HOST_STATUS_CACHE_COUNTER_PATH = Path.fromString("/vespa/host-status-service-cache-counter");
private final CuratorCounter counter;
private final HostInfosService wrappedService;
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java
index dd49b2595a4..2167b45bc02 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/CuratorCounter.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.curator.recipes;
+import com.yahoo.path.Path;
import com.yahoo.vespa.curator.Curator;
import org.apache.curator.framework.recipes.atomic.AtomicValue;
import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
@@ -9,15 +10,14 @@ import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
* A distributed atomic counter.
*
* @author Ulf Lilleengen
- * @since 5.1
*/
public class CuratorCounter {
private final DistributedAtomicLong counter;
- private final String counterPath;
+ private final Path counterPath;
- public CuratorCounter(Curator curator, String counterPath) {
- this.counter = curator.createAtomicCounter(counterPath);
+ public CuratorCounter(Curator curator, Path counterPath) {
+ this.counter = curator.createAtomicCounter(counterPath.getAbsolute());
this.counterPath = counterPath;
}
diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java
index 6b85953a1ff..3d465c6b71f 100644
--- a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java
+++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorCounterTest.java
@@ -1,9 +1,10 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
import org.junit.Test;
+
import static org.junit.Assert.assertEquals;
/**
@@ -14,9 +15,9 @@ public class CuratorCounterTest {
@Test
public void testCounter() throws Exception {
DistributedAtomicLong counter = new MockCurator().createAtomicCounter("/mycounter");
- counter.initialize(4l);
- assertEquals(4l, counter.get().postValue().longValue());
- assertEquals(5l, counter.increment().postValue().longValue());
+ counter.initialize(4L);
+ assertEquals(4L, counter.get().postValue().longValue());
+ assertEquals(5L, counter.increment().postValue().longValue());
}
}