Publish

10 files changed, 643 insertions, 0 deletions

diff --git a/clustercontroller-standalone/.gitignore b/clustercontroller-standalone/.gitignore
new file mode 100644
index 00000000000..3cc25b51fc4
--- /dev/null
+++ b/clustercontroller-standalone/.gitignore
@@ -0,0 +1,2 @@
+/pom.xml.build
+/target
diff --git a/clustercontroller-standalone/OWNERS b/clustercontroller-standalone/OWNERS
new file mode 100644
index 00000000000..b3db17e22d8
--- /dev/null
+++ b/clustercontroller-standalone/OWNERS
@@ -0,0 +1,2 @@
+vekterli
+hakon
diff --git a/clustercontroller-standalone/pom.xml b/clustercontroller-standalone/pom.xml
new file mode 100644
index 00000000000..0569a8976f5
--- /dev/null
+++ b/clustercontroller-standalone/pom.xml
@@ -0,0 +1,108 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>clustercontroller-standalone</artifactId>
+    <version>6-SNAPSHOT</version>
+    <packaging>jar</packaging>
+    <dependencies>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-jdk14</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>com.yahoo.vespa</groupId>
+          <artifactId>configdefinitions</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.yahoo.vespa</groupId>
+          <artifactId>config</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.yahoo.vespa</groupId>
+          <artifactId>vespalog</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>clustercontroller-apps</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>clustercontroller-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>clustercontroller-utils</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-dev</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-runtime</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vdslib</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>zkfacade</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <compilerArgs>
+                        <arg>-Xlint:all</arg>
+                        <arg>-Werror</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly-foo</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcher.java b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcher.java
new file mode 100644
index 00000000000..dbe0c167cfb
--- /dev/null
+++ b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcher.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Used to manually fetch config for the stand alone application.
+ */
+package com.yahoo.vespa.clustercontroller.standalone;
+
+import com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterControllerClusterConfigurer;
+import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions;
+import com.yahoo.vespa.config.content.FleetcontrollerConfig;
+import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.cloud.config.SlobroksConfig;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.config.subscription.ConfigHandle;
+import com.yahoo.cloud.config.ZookeepersConfig;
+
+import java.util.logging.Logger;
+
+public class ClusterControllerConfigFetcher {
+    private static Logger log = Logger.getLogger(ClusterControllerConfigFetcher.class.getName());
+
+    private final ConfigSubscriber configSubscriber = new ConfigSubscriber();
+    private final ConfigHandle<FleetcontrollerConfig> fleetcontrollerConfigHandle;
+    private final ConfigHandle<SlobroksConfig> slobrokConfigHandle;
+    private final ConfigHandle<StorDistributionConfig> distributionConfigHandle;
+    private final ConfigHandle<ZookeepersConfig> zookeeperConfigHandle;
+
+    private FleetControllerOptions options;
+
+    public ClusterControllerConfigFetcher() throws Exception {
+        String configId = createConfigId();
+        log.fine("Using fleetcontroller config id \"" + configId + '"');
+        String slobrokConfigId = createSlobrokConfigId();
+        log.fine("Using slobrok config id \"" + slobrokConfigId + '"');
+
+        fleetcontrollerConfigHandle = configSubscriber.subscribe(FleetcontrollerConfig.class, configId);
+        slobrokConfigHandle = configSubscriber.subscribe(SlobroksConfig.class, slobrokConfigId);
+        distributionConfigHandle = configSubscriber.subscribe(StorDistributionConfig.class, configId);
+        zookeeperConfigHandle = configSubscriber.subscribe(ZookeepersConfig.class, configId);
+
+        if (!configReady()) {
+            throw new IllegalStateException("Initial configuration failed.");
+        }
+        options = generateOptions();
+    }
+
+    public void close() {
+        log.fine("Shutting down fleetcontroller config subscription");
+        configSubscriber.close();
+    }
+
+    private String createConfigId() {
+        return System.getProperty("config.id");
+    }
+
+    private String createSlobrokConfigId() {
+        return System.getProperty("slobrok.config.id");
+    }
+
+    public FleetControllerOptions getOptions() {
+        return options;
+    }
+
+    public FleetControllerOptions generateOptions() throws Exception {
+        ClusterControllerClusterConfigurer configurer = new ClusterControllerClusterConfigurer(
+                null,
+                distributionConfigHandle.getConfig(),
+                fleetcontrollerConfigHandle.getConfig(),
+                slobrokConfigHandle.getConfig(),
+                zookeeperConfigHandle.getConfig(),
+                null);
+        return configurer.getOptions();
+    }
+
+    /** Test to see if the config has been updated, and if so, update the config. */
+    public boolean updated(long timeoutMillis) throws Exception {
+        if (configUpdated(timeoutMillis)) {
+            log.fine("Updated fleetcontroller config.");
+            options = generateOptions();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public long getGeneration() {
+        return configSubscriber.getGeneration();
+    }
+
+    boolean configReady() {
+        return configSubscriber.nextConfig();
+    }
+
+    boolean configUpdated(long timeoutMillis) {
+        return configSubscriber.nextConfig(timeoutMillis);
+    }
+}
diff --git a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java
new file mode 100644
index 00000000000..a07f46ee682
--- /dev/null
+++ b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.standalone;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogSetup;
+import com.yahoo.log.event.Event;
+import com.yahoo.vespa.clustercontroller.core.FleetController;
+
+import java.util.logging.Logger;
+
+/**
+ * This is the class containing the main method used to run fleet controller as a stand-alone program.
+ */
+public class StandAloneClusterController {
+    private static Logger log = Logger.getLogger(StandAloneClusterController.class.getName());
+
+    private final ClusterControllerConfigFetcher config;
+    private FleetController controller;
+    private boolean started = false, stop = false;
+
+    public static class ShutdownHook extends Thread {
+        private StandAloneClusterController app;
+
+        public ShutdownHook(StandAloneClusterController app) {
+            this.app = app;
+        }
+
+        @Override
+        public void run() {
+            try{
+                app.stop();
+            } catch (Exception e) {
+                log.log(LogLevel.FATAL, "Failed to stop application '" + app.getName() + "': " + e.getMessage());
+                e.printStackTrace();
+                return;
+            }
+        }
+
+    }
+
+    public static void main(String args[]) throws Exception {
+        runApplication(new StandAloneClusterController(new ClusterControllerConfigFetcher()));
+    }
+
+    public static void runApplication(StandAloneClusterController myApp) {
+        LogSetup.initVespaLogging("fleetcontroller");
+        try{
+            myApp.start();
+        } catch (Exception e) {
+            log.log(LogLevel.FATAL, "Failed to start application '" + myApp.getName() + "': " + e.getMessage());
+            e.printStackTrace();
+            return;
+        }
+        Runtime.getRuntime().addShutdownHook(new ShutdownHook(myApp));
+        try{
+            myApp.run();
+        } catch (Exception e) {
+            log.log(LogLevel.FATAL, "Application '" + myApp.getName() + "' runtime failure: " + e.getMessage());
+            e.printStackTrace();
+        }
+    }
+
+    public StandAloneClusterController(ClusterControllerConfigFetcher config) {
+        this.config = config;
+    }
+
+    public String getName() {
+        return "Fleetcontroller " + config.getOptions().fleetControllerIndex
+             + " of cluster " + config.getOptions().clusterName;
+    }
+
+    public void start() throws Exception {
+        controller = FleetController.createForStandAlone(config.getOptions());
+        Event.started(getName());
+    }
+
+    public void run() throws Exception {
+        synchronized(this) {
+            started = true;
+        }
+        try{
+            while (true) {
+                synchronized (this) {
+                    if (stop) {
+                        notifyAll();
+                        return;
+                    }
+                    if (config.updated(1)) {
+                        controller.updateOptions(config.getOptions(), config.getGeneration());
+                    }
+                    try{ wait(1000); } catch (InterruptedException e) {}
+                }
+            }
+        } finally {
+            started = false;
+        }
+    }
+
+    public void stop() throws Exception {
+        Event.stopping(getName(), "controlled shutdown");
+        synchronized (this) {
+            controller.shutdown();
+            stop = true;
+            notifyAll();
+            while (started) {
+                try{ wait(1000); } catch (InterruptedException e) {}
+            }
+            config.close();
+        }
+        log.fine("Fleetcontroller done shutting down");
+    }
+
+}
diff --git a/clustercontroller-standalone/src/main/resources/log4j-vespa-fleetcontroller.properties b/clustercontroller-standalone/src/main/resources/log4j-vespa-fleetcontroller.properties
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/clustercontroller-standalone/src/main/resources/log4j-vespa-fleetcontroller.properties
diff --git a/clustercontroller-standalone/src/main/sh/fleetcontroller-start b/clustercontroller-standalone/src/main/sh/fleetcontroller-start
new file mode 100755
index 00000000000..dda006ff7b1
--- /dev/null
+++ b/clustercontroller-standalone/src/main/sh/fleetcontroller-start
@@ -0,0 +1,71 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        # ensure it ends with "/" :
+        VESPA_HOME=${VESPA_HOME%/}/
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+JAR=lib/jars/clustercontroller-standalone-jar-with-dependencies.jar
+
+ROOT=$VESPA_HOME
+export ROOT
+cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; }
+
+jvmargs="-server -Xms16m -Xmx2048m -Dlog4j.configuration=file:$VESPA_HOME/etc/log4j-vespa-fleetcontroller.properties"
+jar="-cp $ROOT/$JAR com.yahoo.vespa.clustercontroller.standalone.StandAloneClusterController"
+exec java $jvmargs -Dconfig.id=$VESPA_CONFIG_ID -Dslobrok.config.id="admin/slobrok.0" "$@" $jar
diff --git a/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcherTest.java b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcherTest.java
new file mode 100644
index 00000000000..32096953b76
--- /dev/null
+++ b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerConfigFetcherTest.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.standalone;
+
+public class ClusterControllerConfigFetcherTest extends ClusterControllerTest {
+    public void testSimple() throws Exception {
+        setFleetControllerConfigProperty();
+        setSlobrokConfigProperty();
+        addFleetControllerConfig(2, 1);
+        addSlobrokConfig();
+        addDistributionConfig();
+        addZookeepersConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher();
+        configFetcher.getOptions();
+        configFetcher.updated(100);
+        assertEquals(1, configFetcher.getGeneration());
+        configFetcher.close();
+    }
+
+    public void testInitialConfigFailure() throws Exception {
+        setFleetControllerConfigProperty();
+        setSlobrokConfigProperty();
+        addFleetControllerConfig(2, 1);
+        addSlobrokConfig();
+        addDistributionConfig();
+        addZookeepersConfig();
+        try{
+            ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher() {
+                boolean configReady() {
+                    return false;
+                }
+            };
+            fail("Control should not reach here");
+        } catch (IllegalStateException e) {
+            assertEquals("Initial configuration failed.", e.getMessage());
+        }
+    }
+
+    public void testConfigUpdate() throws Exception {
+        setFleetControllerConfigProperty();
+        setSlobrokConfigProperty();
+        addFleetControllerConfig(2, 1);
+        addSlobrokConfig();
+        addDistributionConfig();
+        addZookeepersConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher() {
+            boolean configUpdated(long millis) {
+                return true;
+            }
+        };
+        configFetcher.updated(1000);
+    }
+}
diff --git a/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerTest.java b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerTest.java
new file mode 100644
index 00000000000..9b274861e0c
--- /dev/null
+++ b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/ClusterControllerTest.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.standalone;
+
+import com.yahoo.vdslib.distribution.Distribution;
+import junit.framework.TestCase;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+public abstract class ClusterControllerTest extends TestCase {
+    private final Map<String, String> overriddenProperties = new TreeMap<>();
+    private File tempDirectory;
+
+    protected void setProperty(String p, String val) {
+        if (!overriddenProperties.containsKey(p)) {
+            overriddenProperties.put(p, System.getProperty(p));
+        }
+        System.setProperty(p, val);
+    }
+
+    protected File createTemporaryDirectory() throws IOException {
+        File f = File.createTempFile("clustercontroller", "configtest");
+        if (f.exists()) f.delete();
+        f.mkdirs();
+        return f;
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        tempDirectory = createTemporaryDirectory();
+    }
+
+    @Override
+    public void tearDown() {
+        for (Map.Entry<String, String> e : overriddenProperties.entrySet()) {
+            if (e.getValue() == null) {
+                System.clearProperty(e.getKey());
+            } else {
+                System.setProperty(e.getKey(), e.getValue());
+            }
+        }
+        overriddenProperties.clear();
+        if (tempDirectory != null) {
+            for (File f : tempDirectory.listFiles()) {
+                f.delete();
+            }
+            tempDirectory.delete();
+            tempDirectory = null;
+        }
+    }
+
+    protected void writeConfig(String config, String value) throws IOException {
+        File f = new File(tempDirectory, config + ".cfg");
+        FileWriter fw = new FileWriter(f);
+        fw.write(value);
+        fw.close();
+    }
+
+    protected void setFleetControllerConfigProperty() {
+        setProperty("config.id", "dir:" + tempDirectory.toString());
+    }
+
+    protected void addFleetControllerConfig(int stateGatherCount, int fleetcontrollers) throws IOException {
+        writeConfig("fleetcontroller", "cluster_name \"storage\"\n" +
+                "index 0\n" +
+                "state_gather_count " + stateGatherCount + "\n" +
+                "fleet_controller_count " + fleetcontrollers + "\n" +
+                "zookeeper_server \"\"");
+    }
+    protected void setSlobrokConfigProperty() {
+        setProperty("slobrok.config.id", "dir:" + tempDirectory.toString());
+    }
+    protected void addSlobrokConfig() throws IOException {
+        writeConfig("slobroks", "cluster_name \"storage\"\n" +
+                "index 0\n" +
+                "zookeeper_server \"\"");
+    }
+    protected void addDistributionConfig() throws IOException {
+        writeConfig("stor-distribution", Distribution.getDefaultDistributionConfig(2, 10));
+    }
+    protected void addZookeepersConfig() throws IOException {
+        writeConfig("zookeepers", "zookeeperserverlist \"\"");
+    }
+
+    protected void setupConfig() throws Exception {
+        setFleetControllerConfigProperty();
+        setSlobrokConfigProperty();
+        addFleetControllerConfig(2, 1);
+        addSlobrokConfig();
+        addDistributionConfig();
+        addZookeepersConfig();
+    }
+}
diff --git a/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterControllerTest.java b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterControllerTest.java
new file mode 100644
index 00000000000..6f673185a5d
--- /dev/null
+++ b/clustercontroller-standalone/src/test/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterControllerTest.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.standalone;
+
+public class StandAloneClusterControllerTest extends ClusterControllerTest {
+
+    abstract class Runner implements Runnable {
+        boolean completed = false;
+        Exception exception = null;
+
+        public void run() {
+            try{
+                attemptRun();
+                completed = true;
+            } catch (Exception e) {
+                exception = e;
+            }
+        }
+
+        public abstract void attemptRun() throws Exception;
+    }
+
+    public void testSimpleStartStop() throws Exception {
+        setupConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher();
+        StandAloneClusterController controller = new StandAloneClusterController(configFetcher);
+        assertEquals("Fleetcontroller 0 of cluster storage", controller.getName());
+        controller.start();
+        controller.stop();
+    }
+
+    public void testShortRun() throws Exception {
+        setupConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher() {
+            int counter = 0;
+            @Override
+            public boolean updated(long timeoutMillis) throws Exception {
+                return (++counter % 2 == 0);
+            }
+            @Override
+            public void close() {
+                throw new RuntimeException("Failed to stop");
+            }
+        };
+        final StandAloneClusterController controller = new StandAloneClusterController(configFetcher);
+        assertEquals("Fleetcontroller 0 of cluster storage", controller.getName());
+        controller.start();
+        Runner r = new Runner() {
+            @Override
+            public void attemptRun() throws Exception {
+                controller.run();
+            }
+        };
+        Thread t = new Thread(r);
+        t.start();
+        for (int i=0; i<100; ++i) {
+            try{ Thread.sleep(1); } catch (InterruptedException e) {}
+            synchronized (controller) {
+                controller.notifyAll();
+            }
+        }
+        StandAloneClusterController.ShutdownHook hook = new StandAloneClusterController.ShutdownHook(controller);
+        hook.start();
+        hook.join();
+        t.join();
+        assertEquals(true, r.completed);
+        assertNull(r.exception);
+    }
+
+    public void testFailStart() throws Exception {
+        setupConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher();
+        StandAloneClusterController controller = new StandAloneClusterController(configFetcher) {
+            @Override
+            public void start() throws Exception {
+                throw new RuntimeException("Foo");
+            }
+        };
+        StandAloneClusterController.runApplication(controller);
+    }
+
+    public void testFailRun() throws Exception {
+        setupConfig();
+        ClusterControllerConfigFetcher configFetcher = new ClusterControllerConfigFetcher();
+        StandAloneClusterController controller = new StandAloneClusterController(configFetcher) {
+            @Override
+            public void run() throws Exception {
+                throw new RuntimeException("Foo");
+            }
+        };
+        StandAloneClusterController.runApplication(controller);
+    }
+
+    public void testCallMainToGetCoverage() throws Exception {
+        tearDown();
+        setProperty("config.id", "file:non-existing");
+        try{
+            StandAloneClusterController.main(new String[0]);
+            fail("Control should not get here");
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+}