From 20138e8a44b6084b6930fc84f312266866b1c69c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Sun, 20 Oct 2019 19:56:41 +0200 Subject: Revert "Revert "Move ZooKeeperServer to another module"" --- .../com/yahoo/vespa/zookeeper/ZooKeeperServer.java | 132 +++++++++++++++++++ .../com/yahoo/vespa/zookeeper/package-info.java | 5 + .../yahoo/vespa/zookeeper/ZooKeeperServerTest.java | 140 +++++++++++++++++++++ 3 files changed, 277 insertions(+) create mode 100644 zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java create mode 100644 zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/package-info.java create mode 100644 zookeeper-server/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java (limited to 'zookeeper-server') diff --git a/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java new file mode 100644 index 00000000000..252dc6697b0 --- /dev/null +++ b/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java @@ -0,0 +1,132 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.log.LogLevel; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Writes zookeeper config and starts zookeeper server. + * + * @author Ulf Lilleengen + */ +public class ZooKeeperServer extends AbstractComponent implements Runnable { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ZooKeeperServer.class.getName()); + private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; + static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; + private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; + + ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) { + this.zookeeperServerConfig = zookeeperServerConfig; + System.setProperty("zookeeper.jmx.log4j.disable", "true"); + System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); + System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); + + writeConfigToDisk(zookeeperServerConfig); + zkServerThread = new Thread(this, "zookeeper server"); + if (startServer) { + zkServerThread.start(); + } + } + + @Inject + public ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig) { + this(zookeeperServerConfig, true); + } + + private void writeConfigToDisk(ZookeeperServerConfig config) { + String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile()); + new File(configFilePath).getParentFile().mkdirs(); + try (FileWriter writer = new FileWriter(configFilePath)) { + writer.write(transformConfigToString(config)); + writeMyIdFile(config); + } catch (IOException e) { + throw new RuntimeException("Error writing zookeeper config", e); + } + } + + private String transformConfigToString(ZookeeperServerConfig config) { + StringBuilder sb = new StringBuilder(); + sb.append("tickTime=").append(config.tickTime()).append("\n"); + sb.append("initLimit=").append(config.initLimit()).append("\n"); + sb.append("syncLimit=").append(config.syncLimit()).append("\n"); + sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n"); + sb.append("snapCount=").append(config.snapshotCount()).append("\n"); + sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n"); + sb.append("clientPort=").append(config.clientPort()).append("\n"); + sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); + sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); + // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands + // Includes all available commands in 3.4, except 'wchc' and 'wchp' + // Mandatory when using ZooKeeper 3.5 + sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); + ensureThisServerIsRepresented(config.myid(), config.server()); + config.server().forEach(server -> addServerToCfg(sb, server)); + return sb.toString(); + } + + private void writeMyIdFile(ZookeeperServerConfig config) throws IOException { + if (config.server().size() > 1) { + try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) { + writer.write(config.myid() + "\n"); + } + } + } + + private void ensureThisServerIsRepresented(int myid, List servers) { + boolean found = false; + for (ZookeeperServerConfig.Server server : servers) { + if (myid == server.id()) { + found = true; + break; + } + } + if (!found) { + throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")"); + } + } + + private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) { + sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n"); + } + + private void shutdown() { + zkServerThread.interrupt(); + try { + zkServerThread.join(); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e); + } + } + + @Override + public void run() { + System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); + String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; + log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]); + log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); + org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + } + + @Override + public void deconstruct() { + shutdown(); + super.deconstruct(); + } + + private static Set zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { + return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); + } + +} diff --git a/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/package-info.java b/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/package-info.java new file mode 100644 index 00000000000..470d1350f9c --- /dev/null +++ b/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.zookeeper; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zookeeper-server/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java b/zookeeper-server/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java new file mode 100644 index 00000000000..3f33892fd45 --- /dev/null +++ b/zookeeper-server/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java @@ -0,0 +1,140 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * Tests the zookeeper server. + */ +public class ZooKeeperServerTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void config_is_written_correctly_when_one_server() throws IOException { + File cfgFile = folder.newFile(); + File idFile = folder.newFile(); + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.myidFile(idFile.getAbsolutePath()); + builder.server(newServer(0, "foo", 123, 321)); + builder.myid(0); + createServer(builder); + validateConfigFileSingleHost(cfgFile); + validateIdFile(idFile, ""); + } + + @Test + public void config_is_written_correctly_when_multiple_servers() throws IOException { + File cfgFile = folder.newFile(); + File idFile = folder.newFile(); + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.server(newServer(0, "foo", 123, 321)); + builder.server(newServer(1, "bar", 234, 432)); + builder.server(newServer(2, "baz", 345, 543)); + builder.myidFile(idFile.getAbsolutePath()); + builder.myid(1); + createServer(builder); + validateConfigFileMultipleHosts(cfgFile); + validateIdFile(idFile, "1\n"); + } + + private void createServer(ZookeeperServerConfig.Builder builder) { + new ZooKeeperServer(new ZookeeperServerConfig(builder), false); + } + + @Test(expected = RuntimeException.class) + public void require_that_this_id_must_be_present_amongst_servers() { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.server(newServer(1, "bar", 234, 432)); + builder.server(newServer(2, "baz", 345, 543)); + builder.myid(0); + createServer(builder); + } + + @Test + public void juteMaxBufferCanBeSet() throws IOException { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.myid(0); + File idFile = folder.newFile(); + File cfgFile = folder.newFile(); + + builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost")); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.myidFile(idFile.getAbsolutePath()); + + createServer(builder); + assertThat(System.getProperty(ZooKeeperServer.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); + + final int max_buffer = 1; + builder.juteMaxBuffer(max_buffer); + createServer(builder); + assertThat(System.getProperty(ZooKeeperServer.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); + } + + private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) { + ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); + builder.id(id); + builder.hostname(hostName); + builder.electionPort(electionPort); + builder.quorumPort(quorumPort); + return builder; + } + + private void validateIdFile(File idFile, String expected) throws IOException { + String actual = IOUtils.readFile(idFile); + assertThat(actual, is(expected)); + } + + private void validateConfigFileSingleHost(File cfgFile) throws IOException { + String expected = + "tickTime=2000\n" + + "initLimit=20\n" + + "syncLimit=15\n" + + "maxClientCnxns=0\n" + + "snapCount=50000\n" + + "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + + "clientPort=2181\n" + + "autopurge.purgeInterval=1\n" + + "autopurge.snapRetainCount=15\n" + + "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + + "server.0=foo:321:123\n"; + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { + String expected = + "tickTime=2000\n" + + "initLimit=20\n" + + "syncLimit=15\n" + + "maxClientCnxns=0\n" + + "snapCount=50000\n" + + "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + + "clientPort=2181\n" + + "autopurge.purgeInterval=1\n" + + "autopurge.snapRetainCount=15\n" + + "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + + "server.0=foo:321:123\n" + + "server.1=bar:432:234\n" + + "server.2=baz:543:345\n"; + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFile(File cfgFile, String expected) throws IOException { + String actual = IOUtils.readFile(cfgFile); + assertThat(actual, is(expected)); + } +} -- cgit v1.2.3