Publish

85 files changed, 4174 insertions, 0 deletions

diff --git a/clustercontroller-utils/.gitignore b/clustercontroller-utils/.gitignore
new file mode 100644
index 00000000000..12251442258
--- /dev/null
+++ b/clustercontroller-utils/.gitignore
@@ -0,0 +1,2 @@
+/target
+/pom.xml.build
diff --git a/clustercontroller-utils/OWNERS b/clustercontroller-utils/OWNERS
new file mode 100644
index 00000000000..b3db17e22d8
--- /dev/null
+++ b/clustercontroller-utils/OWNERS
@@ -0,0 +1,2 @@
+vekterli
+hakon
diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml
new file mode 100644
index 00000000000..10d6c220d86
--- /dev/null
+++ b/clustercontroller-utils/pom.xml
@@ -0,0 +1,63 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>clustercontroller-utils</artifactId>
+    <version>6-SNAPSHOT</version>
+    <packaging>container-plugin</packaging>
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.codehaus.jettison</groupId>
+            <artifactId>jettison</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>annotations</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespalog</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>yolean</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
+                <extensions>true</extensions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <compilerArgs>
+                        <arg>-Xlint:unchecked</arg>
+                        <arg>-Xlint:deprecation</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java
new file mode 100644
index 00000000000..a68b90b6eef
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public interface AsyncCallback<T>   {
+    /** Callback indicating the given operation has completed. */
+    public void done(AsyncOperation<T> op);
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java
new file mode 100644
index 00000000000..3099f57ba09
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public interface AsyncOperation<T>   {
+    /**
+     * Attempt to cancel the given operation.
+     * @return True if successfully cancelled. False if cancel is not supported, or operation already succeeded.
+     */
+    public boolean cancel();
+
+    /**
+     * Register a callback to be called when this operation completes. If operation is already completed, this callback
+     * will be called immediately upon registering. The same callback should not be registered multiple times. It is
+     * suggested to throw an exception if that should happen. Otherwise you may get one or more calls to that component.
+     */
+    void register(AsyncCallback<T> callback);
+
+    /**
+     * Remove a callback from the list to be called when operation is completed. If callback has not already been called
+     * at the time this function returns, it should never be called by this operation, unless re-registered.
+     */
+    void unregister(AsyncCallback<T> callback);
+
+    /**
+     * Get the name of the operation. Useful to identify what operation this is.
+     */
+    public String getName();
+
+    /**
+     * Get a description of the operation. May be empty. If operation is complex one might want to use a short name for
+     * simplicity, but have the whole request available if needed. In the HTTP case an application may for instance include
+     * the URL in the name, and add the request headers to the description.
+     */
+    public String getDescription();
+
+    /**
+     * Get the progress as a number between 0 and 1 where 0 means not started and 1 means operation is complete.
+     * A return value of null indicates that the operation is unable to track progress.
+     */
+    public Double getProgress();
+
+    /**
+     * Get the result of the operation.
+     * Note that some operations may not have a result if the operation failed.
+     */
+    public T getResult();
+
+    /** Get the cause of an operation failing. Returns null on successful operations. */
+    public Exception getCause();
+
+    /** Returns true if operation has been successfully cancelled. */
+    public boolean isCanceled();
+    /** Returns true if operation has completed. Regardless of whether it was a success or a failure. */
+    public boolean isDone();
+    /** Returns true if the operation was a success. */
+    public boolean isSuccess();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java
new file mode 100644
index 00000000000..79da91d40b2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import java.util.logging.Logger;
+
+public class AsyncOperationImpl<T> implements AsyncOperation<T> {
+    private static final Logger log = Logger.getLogger(AsyncOperationImpl.class.getName());
+    private final String name;
+    private final String description;
+    private boolean resultGotten = false; // Ensures result is set only once.
+    private boolean completed = false; // Indicates that operation is complete.
+    private boolean failed = false;
+    private T result;
+    private Exception failure;
+    private AsyncOperationListenImpl<T> listenImpl;
+
+    public AsyncOperationImpl(String name) {
+        this(name, null);
+    }
+    public AsyncOperationImpl(String name, String description) {
+        this.name = name;
+        this.description = description;
+        listenImpl = new AsyncOperationListenImpl<T>(this);
+    }
+
+    private boolean tagResultHandled() {
+        synchronized (listenImpl) {
+            if (resultGotten) {
+                log.fine("Operation " + this + " got result attempted set twice. This may occasionally happen if multiple "
+                       + "sources are set to possibly terminate operations, such as for example if there is a separate cancel or timeout "
+                       + "handler.");
+                return false;
+            }
+            resultGotten = true;
+            return true;
+        }
+    }
+
+    public void setFailure(Exception e) { setFailure(e, null); }
+    public void setFailure(Exception e, T partialResult) {
+        if (!tagResultHandled()) { return; }
+        failed = true;
+        failure = e;
+        this.result = partialResult;
+        completed = true;
+        listenImpl.notifyListeners();
+    }
+    public void setResult(T result) {
+        if (!tagResultHandled()) return;
+        this.result = result;
+        completed = true;
+        listenImpl.notifyListeners();
+    }
+
+    @Override
+    public String getName() { return name; }
+    @Override
+    public String getDescription() { return description; }
+    @Override
+    public String toString() { return "AsyncOperationImpl(" + name + ")"; }
+    @Override
+    public T getResult() { return result; }
+    @Override
+    public boolean cancel() { return false; }
+    @Override
+    public boolean isCanceled() { return false; }
+    @Override
+    public boolean isDone() { return completed; }
+    @Override
+    public boolean isSuccess() { return (completed && !failed); }
+    @Override
+    public Double getProgress() { return (completed ? 1.0 : null); }
+    @Override
+    public Exception getCause() { return failure; }
+    @Override
+    public void register(AsyncCallback<T> callback) {
+        listenImpl.register(callback);
+    }
+    @Override
+    public void unregister(AsyncCallback<T> callback) {
+        listenImpl.unregister(callback);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java
new file mode 100644
index 00000000000..e9007f0c16e
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.logging.Logger;
+
+public class AsyncOperationListenImpl<T> {
+    private static final Logger log = Logger.getLogger(AsyncOperationListenImpl.class.getName());
+    private final Collection<AsyncCallback<T>> listeners = new HashSet<>();
+    private boolean listenersNotified = false;
+    private AsyncOperation<T> op;
+
+    protected AsyncOperationListenImpl(AsyncOperation<T> op) {
+        this.op = op;
+    }
+
+    public void register(AsyncCallback<T> callback) {
+        synchronized (listeners) {
+            listeners.add(callback);
+            if (listenersNotified) callback.done(op);
+        }
+    }
+    public void unregister(AsyncCallback<T> callback) {
+        synchronized (listeners) {
+            listeners.remove(callback);
+        }
+    }
+
+    public void notifyListeners() {
+        synchronized (listeners) {
+            if (listenersNotified) return;
+            for(AsyncCallback<T> callback : listeners) {
+                try{
+                    callback.done(op);
+                } catch (RuntimeException e) {
+                    StringWriter sw = new StringWriter();
+                    e.printStackTrace(new PrintWriter(sw));
+                    log.warning("Callback '" + callback + "' threw exception on notify. Should not happen:\n" + sw);
+                }
+            }
+            listenersNotified = true;
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java
new file mode 100644
index 00000000000..71d6f534a9a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public class AsyncUtils {
+
+    public static void waitFor(AsyncOperation op) {
+        while (!op.isDone()) {
+            try{ Thread.sleep(1); } catch (InterruptedException e) {}
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java
new file mode 100644
index 00000000000..a7ce5bb61a2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public abstract class PipedAsyncOperation<S, T> extends RedirectedAsyncOperation<S, T> {
+    private T result;
+
+    public PipedAsyncOperation(AsyncOperation<S> source) {
+        super(source);
+        setOnCompleteTask(new AsyncCallback<S>() {
+            @Override
+            public void done(AsyncOperation<S> op) {
+                result = convertResult(op.getResult());
+            }
+        });
+    }
+
+    public abstract T convertResult(S result);
+
+    @Override
+    public T getResult() {
+        return result;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java
new file mode 100644
index 00000000000..fdaf0dcbf2b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+/**
+ * Utility class in order to wrap com.yahoo.vespa.clustercontroller.utils.communication.async operation callbacks. Useful when translating com.yahoo.vespa.clustercontroller.utils.communication.async operations returning JSON to com.yahoo.vespa.clustercontroller.utils.communication.async operations returning specific values.
+ */
+public abstract class RedirectedAsyncOperation<S, T> implements AsyncOperation<T> {
+    protected final AsyncOperation<S> source;
+    private final AsyncOperationListenImpl<T> listenImpl;
+    private AsyncCallback<S> beforeCallbackTask;
+
+    public RedirectedAsyncOperation(AsyncOperation<S> source) {
+        this.source = source;
+        this.listenImpl = new AsyncOperationListenImpl<>(this);
+        source.register(new AsyncCallback<S>() {
+            @Override
+            public void done(AsyncOperation<S> op) { notifyDone(); }
+        });
+    }
+
+    public RedirectedAsyncOperation<S, T> setOnCompleteTask(AsyncCallback<S> beforeTask) {
+        beforeCallbackTask = beforeTask;
+        return this;
+    }
+    private void notifyDone() {
+        if (beforeCallbackTask != null) beforeCallbackTask.done(source);
+        listenImpl.notifyListeners();
+    }
+
+    @Override
+    public String getName() { return source.getName(); }
+
+    @Override
+    public String getDescription() { return source.getDescription(); }
+
+    @Override
+    public boolean cancel() { return source.cancel(); }
+
+    @Override
+    public boolean isCanceled() { return source.isCanceled(); }
+
+    @Override
+    public boolean isDone() { return source.isDone(); }
+
+    @Override
+    public boolean isSuccess() { return source.isSuccess(); }
+
+    @Override
+    public Double getProgress() { return source.getProgress(); }
+
+    @Override
+    public Exception getCause() { return source.getCause(); }
+
+    @Override
+    public void register(AsyncCallback<T> callback) {
+        listenImpl.register(callback);
+    }
+
+    @Override
+    public void unregister(AsyncCallback<T> callback) {
+        listenImpl.unregister(callback);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java
new file mode 100644
index 00000000000..0572465aac5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public abstract class SuccessfulAsyncCallback<Source, Target> implements AsyncCallback<Source> {
+    private final AsyncOperationImpl<Target> target;
+
+    public SuccessfulAsyncCallback(final AsyncOperationImpl<Target> target) {
+        this.target = target;
+    }
+
+    public void done(AsyncOperation<Source> sourceOp) {
+        if (sourceOp.isSuccess()) {
+            successfullyDone(sourceOp);
+        } else {
+            target.setFailure(sourceOp.getCause());
+        }
+    }
+
+    public abstract void successfullyDone(AsyncOperation<Source> op);
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java
new file mode 100644
index 00000000000..205da8cf98f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java
new file mode 100644
index 00000000000..5a5dd21c8f5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+/**
+ * Abstraction of an asynchronious HTTP client, such that applications don't need to depend directly on an HTTP client.
+ */
+public interface AsyncHttpClient<V extends HttpResult> {
+
+    public AsyncOperation<V> execute(HttpRequest r);
+
+    /** Attempt to cancel all pending operations and shut down the client. */
+    public void close();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java
new file mode 100644
index 00000000000..419c19f529b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+public class AsyncHttpClientWithBase<V extends HttpResult> implements AsyncHttpClient<V> {
+    protected final AsyncHttpClient<V> client;
+    private HttpRequest baseRequest = new HttpRequest();
+
+    public AsyncHttpClientWithBase(AsyncHttpClient<V> client) {
+        if (client == null) throw new IllegalArgumentException("HTTP client must be set.");
+        this.client = client;
+    }
+
+    /**
+     * If all your http requests have common features you want to set once, you can provide those values in a base
+     * request. For instance, if you specify a host and a port using this function, all your requests will use that
+     * host and port unless specified in the request you execute.
+     */
+    public void setHttpRequestBase(HttpRequest r) {
+        this.baseRequest = (r == null ? new HttpRequest() : r.clone());
+    }
+
+    public HttpRequest getHttpRequestBase() {
+        return baseRequest;
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        return client.execute(baseRequest.merge(r));
+    }
+
+    @Override
+    public void close() {
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java
new file mode 100644
index 00000000000..c64da81cac1
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.util.CertainlyCloneable;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class HttpRequest extends CertainlyCloneable<HttpRequest> {
+    public static class KeyValuePair {
+        public String key;
+        public String value;
+
+        public KeyValuePair(String k, String v) { this.key = k; this.value = v; }
+
+        public String getValue() {
+            return value;
+        }
+
+        public String getKey() {
+            return key;
+        }
+    }
+    public enum HttpOp { GET, POST, PUT, DELETE }
+
+    private String host;
+    private int port;
+    private String path;
+    private List<KeyValuePair> urlOptions = new LinkedList<>();
+    private List<KeyValuePair> headers = new LinkedList<>();
+    private long timeoutMillis;
+    private Object postContent;
+    private HttpOp httpOperation;
+
+    public HttpRequest() {}
+
+    public String getHost() { return host; }
+    public int getPort() { return port; }
+    public String getPath() { return path; }
+    public List<KeyValuePair> getUrlOptions() { return urlOptions; }
+    public String getOption(String key, String defaultValue) {
+        for (KeyValuePair value : urlOptions) {
+            if (value.key.equals(key)) return value.value;
+        }
+        return defaultValue;
+
+    }
+    public List<KeyValuePair> getHeaders() { return headers; }
+    public String getHeader(String key, String defaultValue) {
+        for (KeyValuePair header : headers) {
+            if (header.key.equals(key)) return header.value;
+        }
+        return defaultValue;
+    }
+    public long getTimeoutMillis() { return timeoutMillis; }
+    public Object getPostContent() { return postContent; }
+    public HttpOp getHttpOperation() { return httpOperation; }
+
+    public HttpRequest setHost(String hostname) { this.host = hostname; return this; }
+    public HttpRequest setPort(int port) { this.port = port; return this; }
+    public HttpRequest setPath(String path) {
+        this.path = path; return this;
+    }
+    public HttpRequest addUrlOption(String key, String value) { this.urlOptions.add(new KeyValuePair(key, value)); return this; }
+    public HttpRequest setUrlOptions(List<KeyValuePair> options) { this.urlOptions.clear(); this.urlOptions.addAll(options); return this; }
+    public HttpRequest addHttpHeader(String key, String value) { this.headers.add(new KeyValuePair(key, value)); return this; }
+    public HttpRequest setTimeout(long timeoutMillis) { this.timeoutMillis = timeoutMillis; return this; }
+    public HttpRequest setPostContent(Object content) { this.postContent = content; return this; }
+    public HttpRequest setHttpOperation(HttpOp op) { this.httpOperation = op; return this; }
+
+    /** Create a copy of this request, and override what is specified in the input in the new request. */
+    public HttpRequest merge(HttpRequest r) {
+        HttpRequest copy = clone();
+        if (r.host != null) copy.host = r.host;
+        if (r.port != 0) copy.port = r.port;
+        if (r.path != null) copy.path = r.path;
+        for (KeyValuePair h : r.headers) {
+            boolean containsElement = false;
+            for (KeyValuePair h2 : copy.headers) { containsElement |= (h.key.equals(h2.key)); }
+            if (!containsElement) copy.headers.add(h);
+        }
+        for (KeyValuePair h : r.urlOptions) {
+            boolean containsElement = false;
+            for (KeyValuePair h2 : copy.urlOptions) { containsElement |= (h.key.equals(h2.key)); }
+            if (!containsElement) copy.urlOptions.add(h);
+        }
+        if (r.timeoutMillis != 0) copy.timeoutMillis = r.timeoutMillis;
+        if (r.postContent != null) copy.postContent = r.postContent;
+        if (r.httpOperation != null) copy.httpOperation = r.httpOperation;
+        return copy;
+    }
+
+    @Override
+    public HttpRequest clone() {
+        HttpRequest r = (HttpRequest) super.clone();
+        r.headers = new LinkedList<>(r.headers);
+        r.urlOptions = new LinkedList<>(r.urlOptions);
+        return r;
+    }
+
+    @Override
+    public String toString() { return toString(false); }
+    public String toString(boolean verbose) {
+        String httpOp = (httpOperation != null ? httpOperation.toString()
+                                               : (postContent == null ? "GET?" : "POST?"));
+        StringBuilder sb = new StringBuilder().append(httpOp).append(" http:");
+        if (host != null) {
+            sb.append("//").append(host);
+            if (port != 0) sb.append(':').append(port);
+        }
+        if (path == null || path.isEmpty()) {
+            sb.append('/');
+        } else {
+            if (path.charAt(0) != '/') sb.append('/');
+            sb.append(path);
+        }
+        if (urlOptions != null && urlOptions.size() > 0) {
+            boolean first = (path == null || path.indexOf('?') < 0);
+            for (KeyValuePair e : urlOptions) {
+                sb.append(first ? '?' : '&');
+                first = false;
+                sb.append(e.key).append('=').append(e.value);
+            }
+        }
+        if (verbose) {
+            for (KeyValuePair p : headers) {
+                sb.append('\n').append(p.key).append(": ").append(p.value);
+            }
+            if (postContent != null && !postContent.toString().isEmpty()) {
+                sb.append("\n\n").append(postContent.toString());
+            }
+        }
+        return sb.toString();
+    }
+
+    public void verifyComplete() {
+        if (path == null) throw new IllegalStateException("HTTP requests must have a path set. Use '/' for top level");
+        if (httpOperation == null) throw new IllegalStateException("HTTP requests must have an HTTP method defined");
+    }
+
+    public boolean equals(Object o) {
+        // Equals is only used for debugging as far as we know. Refer to verbose toString to simplify
+        if (o instanceof HttpRequest) {
+            return toString(true).equals(((HttpRequest) o).toString(true));
+        }
+        return false;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java
new file mode 100644
index 00000000000..b8acfa33e71
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+public interface HttpRequestHandler {
+
+    public HttpResult handleRequest(HttpRequest request) throws Exception;
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java
new file mode 100644
index 00000000000..7e2bf90b92c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+public class HttpResult {
+    private static class HttpReturnCode {
+        private final int code;
+        private final String message;
+
+        public HttpReturnCode(int code, String message) {
+            this.code = code;
+            this.message = message;
+        }
+        public boolean isSuccess() { return (code >= 200 && code < 300); }
+        public int getCode() { return code; }
+        public String getMessage() { return message; }
+    }
+    private HttpReturnCode httpReturnCode;
+    private final List<HttpRequest.KeyValuePair> headers;
+    private Object content;
+
+    public HttpResult() {
+        httpReturnCode = new HttpReturnCode(200, "OK");
+        headers = new LinkedList<>();
+    }
+
+    public HttpResult(HttpResult other) {
+        httpReturnCode = other.httpReturnCode;
+        headers = other.headers;
+        content = other.content;
+    }
+
+    public HttpResult setHttpCode(int code, String description) {
+        this.httpReturnCode = new HttpReturnCode(code, description);
+        return this;
+    }
+
+    public HttpResult setContent(Object content) {
+        this.content = content;
+        return this;
+    }
+
+    public HttpResult addHeader(String key, String value) {
+        headers.add(new HttpRequest.KeyValuePair(key, value));
+        return this;
+    }
+
+    public boolean isSuccess() { return httpReturnCode.isSuccess(); }
+    public int getHttpReturnCode() { return httpReturnCode.getCode(); }
+    public String getHttpReturnCodeDescription() { return httpReturnCode.getMessage(); }
+    public Collection<HttpRequest.KeyValuePair> getHeaders() { return headers; }
+    public String getHeader(String key) {
+        for (HttpRequest.KeyValuePair p : headers) {
+            if (p.getKey().equals(key)) return p.getValue();
+        }
+        return null;
+    }
+
+    public Object getContent() { return content; }
+
+    @Override
+    public String toString() { return toString(false); }
+    public String toString(boolean verbose) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("HTTP ").append(httpReturnCode.getCode()).append('/').append(httpReturnCode.getMessage());
+        if (verbose) {
+            for(HttpRequest.KeyValuePair header : headers) {
+                sb.append('\n').append(header.getKey()).append(": ").append(header.getValue());
+            }
+            if (content != null) {
+                StringBuilder contentBuilder = new StringBuilder();
+                printContent(contentBuilder);
+                String s = contentBuilder.toString();
+                if (!s.isEmpty()) {
+                    sb.append("\n\n").append(s);
+                }
+            }
+        }
+        return sb.toString();
+    }
+    public void printContent(StringBuilder sb) {
+        sb.append(content.toString());
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java
new file mode 100644
index 00000000000..bffac7f68e3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.RedirectedAsyncOperation;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * Wrapped for the HTTP client, converting requests to/from JSON.
+ */
+public class JsonAsyncHttpClient implements AsyncHttpClient<JsonHttpResult> {
+    private AsyncHttpClient<HttpResult> client;
+    private boolean verifyRequestContentAsJson = true;
+    private boolean addJsonContentType = true;
+
+    public JsonAsyncHttpClient(AsyncHttpClient<HttpResult> client) {
+        this.client = client;
+    }
+
+    public JsonAsyncHttpClient verifyRequestContentAsJson(boolean doIt) {
+        verifyRequestContentAsJson = doIt;
+        return this;
+    }
+
+    public JsonAsyncHttpClient addJsonContentType(boolean doIt) {
+        addJsonContentType = doIt;
+        return this;
+    }
+
+    public AsyncOperation<JsonHttpResult> execute(HttpRequest r) {
+        if (verifyRequestContentAsJson) {
+            if (r.getPostContent() != null && !(r.getPostContent() instanceof JSONObject)) {
+                try{
+                    r = r.clone().setPostContent(new JSONObject(r.getPostContent().toString()));
+                } catch (JSONException e) {
+                    throw new IllegalArgumentException(e);
+                }
+            }
+        }
+        if (addJsonContentType && r.getPostContent() != null) {
+            r = r.clone().addHttpHeader("Content-Type", "application/json");
+        }
+        final AsyncOperation<HttpResult> op = client.execute(r);
+        return new RedirectedAsyncOperation<HttpResult, JsonHttpResult>(op) {
+            @Override
+            public JsonHttpResult getResult() {
+                return (op.getResult() == null ? null : new JsonHttpResult(op.getResult()));
+            }
+        };
+    }
+
+    @Override
+    public void close() {
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java
new file mode 100644
index 00000000000..e26ea4fc3a2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.util.JSONObjectWrapper;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonHttpResult extends HttpResult {
+    private JSONObject json;
+    private boolean failedParsing = false;
+
+
+    public JsonHttpResult() {
+        addHeader("Content-Type", "application/json");
+    }
+
+    public JsonHttpResult(HttpResult other) {
+        super(other);
+
+        if (other.getContent() == null) {
+            setParsedJson(new JSONObject());
+            return;
+        }
+        try{
+            if (other.getContent() instanceof JSONObject) {
+                setParsedJson((JSONObject) other.getContent());
+            } else {
+                setParsedJson(new JSONObject(other.getContent().toString()));
+            }
+        } catch (JSONException e) {
+            failedParsing = true;
+            setParsedJson(createErrorJson(e.getMessage(), other));
+        }
+    }
+
+    private JSONObject createErrorJson(String error, HttpResult other) {
+        return new JSONObjectWrapper()
+                .put("error", "Invalid JSON in output: " + error)
+                .put("output", other.getContent().toString());
+    }
+
+    public JsonHttpResult setJson(JSONObject o) {
+        setContent(o);
+        json = o;
+        return this;
+    }
+
+    private void setParsedJson(JSONObject o) {
+        json = o;
+    }
+
+    public JSONObject getJson() {
+        return json;
+    }
+
+    @Override
+    public void printContent(StringBuilder sb) {
+        if (failedParsing) {
+            super.printContent(sb);
+            return;
+        }
+        if (json != null) {
+            sb.append("JSON: ");
+            try{
+                sb.append(json.toString(2));
+            } catch (JSONException e) {
+                sb.append(json.toString());
+            }
+        } else {
+            super.printContent(sb);
+        }
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java
new file mode 100644
index 00000000000..d4aff12146d
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.PipedAsyncOperation;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class LoggingAsyncHttpClient<T extends HttpResult> extends AsyncHttpClientWithBase<T> {
+    private static final Logger log = Logger.getLogger(LoggingAsyncHttpClient.class.getName());
+    private int requestCounter = 0;
+
+    public LoggingAsyncHttpClient(AsyncHttpClient<T> client) {
+        super(client);
+        log.info("Logging HTTP requests if fine logging level is added");
+    }
+
+    public AsyncOperation<T> execute(HttpRequest r) {
+        final int requestCount = ++requestCounter;
+        log.fine("Issuing HTTP request " + requestCount + ": " + r.toString(true));
+        final AsyncOperation<T> op = client.execute(r);
+        return new PipedAsyncOperation<T, T>(op) {
+            @Override
+            public T convertResult(T result) {
+                if (log.isLoggable(Level.FINE)) {
+                    if (op.isSuccess()) {
+                        log.fine("HTTP request " + requestCount + " completed: " + result.toString(true));
+                    } else {
+                        StringWriter sw = new StringWriter();
+                        op.getCause().printStackTrace(new PrintWriter(sw));
+                        log.fine("HTTP request " + requestCount + " failed: " + sw);
+                    }
+                }
+                return result;
+            }
+        };
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java
new file mode 100644
index 00000000000..6b63027d0af
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+public class ProxyAsyncHttpClient<V extends HttpResult> extends AsyncHttpClientWithBase<V> {
+    private final String proxyHost;
+    private final int proxyPort;
+
+    public ProxyAsyncHttpClient(AsyncHttpClient<V> client, String proxyHost, int proxyPort) {
+        super(client);
+        this.proxyHost = proxyHost;
+        this.proxyPort = proxyPort;
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        r = getHttpRequestBase().merge(r);
+        if (r.getHost() == null || r.getPath() == null) {
+            throw new IllegalStateException("Host and path must be set prior to being able to proxy an HTTP request");
+        }
+        StringBuilder path = new StringBuilder().append(r.getHost());
+        if (r.getPort() != 0) path.append(':').append(r.getPort());
+        if (r.getPath().isEmpty() || r.getPath().charAt(0) != '/') path.append('/');
+        path.append(r.getPath());
+        return client.execute(r.setHost(proxyHost).setPort(proxyPort).setPath(path.toString()));
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java
new file mode 100644
index 00000000000..6388d32f98d
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+import java.util.LinkedList;
+import java.util.logging.Logger;
+
+/**
+ * Utility class to schedule HTTP requests and keeping a maximum amount of them pending at a time.
+ */
+public class RequestQueue<V extends HttpResult> {
+    private static final Logger log = Logger.getLogger(RequestQueue.class.getName());
+    private final AsyncHttpClient<V> httpClient;
+    private final LinkedList<Request<V>> requestQueue = new LinkedList<>();
+    private final int maxPendingRequests;
+    private int pendingRequests = 0;
+
+    public RequestQueue(AsyncHttpClient<V> httpClient, int maxPendingRequests) {
+        this.httpClient = httpClient;
+        this.maxPendingRequests = maxPendingRequests;
+    }
+
+    public boolean empty() {
+        synchronized (requestQueue) {
+            return (requestQueue.isEmpty() && pendingRequests == 0);
+        }
+    }
+
+    public void waitUntilEmpty() throws InterruptedException {
+        synchronized (requestQueue) {
+            while (!empty()) {
+                requestQueue.wait();
+            }
+        }
+    }
+
+    public void schedule(HttpRequest request, AsyncCallback<V> callback) {
+        log.fine("Scheduling " + request + " call");
+        synchronized (requestQueue) {
+            requestQueue.addLast(new Request<>(request, callback));
+            sendMore();
+        }
+    }
+
+    private void sendMore() {
+        while (pendingRequests < maxPendingRequests && !requestQueue.isEmpty()) {
+            Request<V> call = requestQueue.removeFirst();
+            log.fine("Sending " + call.getRequest() + ".");
+            ++pendingRequests;
+            AsyncOperation<V> op = httpClient.execute(call.getRequest());
+            op.register(call);
+        }
+    }
+
+    private class Request<V extends HttpResult> implements AsyncCallback<V> {
+        private final HttpRequest request;
+        private final AsyncCallback<V> callback;
+
+        Request(HttpRequest request, AsyncCallback<V> callback) {
+            this.request = request;
+            this.callback = callback;
+        }
+
+        public HttpRequest getRequest() { return request; }
+
+        @Override
+        public void done(AsyncOperation<V> op) {
+            if (op.isSuccess()) {
+                log.fine("Operation " + op.getName() + " completed successfully");
+            } else {
+                log.fine("Operation " + op.getName() + " failed: " + op.getCause());
+            }
+            synchronized (requestQueue) {
+                --pendingRequests;
+            }
+            callback.done(op);
+            synchronized (requestQueue) {
+                requestQueue.notifyAll();
+                sendMore();
+            }
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java
new file mode 100644
index 00000000000..cdb20be3338
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+public interface SyncHttpClient {
+    public HttpResult execute(HttpRequest r);
+
+    /** Attempt to cancel all pending operations and shut down the client. */
+    public void close();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java
new file mode 100644
index 00000000000..c406dafe063
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.util.Clock;
+
+import java.util.*;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+
+public class TimeoutHandler<V extends HttpResult> extends AsyncHttpClientWithBase<V> {
+    public static class InternalRequest<V extends HttpResult> extends AsyncOperationImpl<V> {
+        final AsyncOperation<V> operation;
+        long startTime;
+        long timeout;
+
+        public InternalRequest(AsyncOperation<V> op, long startTime, long timeout) {
+            super(op.getName(), op.getDescription());
+            this.operation = op;
+            this.startTime = startTime;
+            this.timeout = timeout;
+            op.register(new AsyncCallback<V>() {
+                @Override
+                public void done(AsyncOperation<V> op) {
+                    if (!isDone()) {
+                        if (op.isSuccess()) {
+                            setResult(op.getResult());
+                        } else {
+                            setFailure(op.getCause(), op.getResult());
+                        }
+                    }
+                }
+            });
+        }
+
+        public long getTimeoutTime() { return startTime + timeout; }
+
+        public void handleTimeout(long currentTime) {
+            long timePassed = currentTime - startTime;
+            this.setFailure(new TimeoutException("Operation timeout. " + timePassed + " ms since operation was issued. Timeout was " + timeout + " ms."));
+            operation.cancel();
+        }
+
+        @Override
+        public boolean cancel() { return operation.cancel(); }
+        @Override
+        public boolean isCanceled() { return operation.isCanceled(); }
+        @Override
+        public Double getProgress() { return (isDone() ? Double.valueOf(1.0) : operation.getProgress()); }
+    }
+
+    public static class ChangeLogger {
+        private InternalRequest lastTimeoutLogged = null;
+        private boolean emptyLogged = true;
+
+        public void logChanges(TreeMap<Long, InternalRequest> requests) {
+            if (requests.isEmpty()) {
+                if (!emptyLogged) {
+                    log.finest("No more pending requests currently.");
+                    emptyLogged = true;
+                }
+            } else {
+                emptyLogged = false;
+                InternalRequest r = requests.firstEntry().getValue();
+                if (lastTimeoutLogged == null || !lastTimeoutLogged.equals(r)) {
+                    lastTimeoutLogged = r;
+                    log.finest("Next operation to possibly timeout will do so at " + r.getTimeoutTime());
+                }
+            }
+        }
+    }
+
+    private final static Logger log = Logger.getLogger(TimeoutHandler.class.getName());
+    private final TreeMap<Long, InternalRequest> requests = new TreeMap<>();
+    private final ChangeLogger changeLogger = new ChangeLogger();
+    private final Clock clock;
+    private boolean run = true;
+    private Runnable timeoutHandler = new Runnable() {
+        @Override
+        public void run() {
+            log.fine("Starting timeout monitor thread");
+            while (true) {
+                performTimeoutHandlerTick();
+                synchronized (clock) {
+                    try{ clock.wait(100); } catch (InterruptedException e) {}
+                    if (!run) break;
+                }
+            }
+            log.fine("Stopped timeout monitor thread");
+        }
+    };
+
+    public TimeoutHandler(Executor executor, Clock clock, AsyncHttpClient<V> client) {
+        super(client);
+        this.clock = clock;
+        executor.execute(timeoutHandler);
+    }
+
+    @Override
+    public void close() {
+        synchronized (clock) {
+            run = false;
+            clock.notifyAll();
+        }
+        synchronized (requests) {
+            for (InternalRequest r : requests.values()) {
+                r.operation.cancel();
+                r.setFailure(new TimeoutException("Timeout handler shutting down. Shutting down all requests monitored."));
+            }
+            requests.clear();
+        }
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        AsyncOperation<V> op = super.execute(r);
+        InternalRequest<V> request = new InternalRequest<>(op, clock.getTimeInMillis(), r.getTimeoutMillis());
+        synchronized (requests) {
+            requests.put(request.getTimeoutTime(), request);
+        }
+        return request;
+    }
+
+    void performTimeoutHandlerTick() {
+        synchronized (requests) {
+            removeCompletedRequestsFromTimeoutList();
+            handleTimeoutsAtTime(clock.getTimeInMillis());
+            changeLogger.logChanges(requests);
+        }
+    }
+
+    private void removeCompletedRequestsFromTimeoutList() {
+        while (!requests.isEmpty() && requests.firstEntry().getValue().operation.isDone()) {
+            requests.remove(requests.firstEntry().getKey());
+            log.finest("Removed completed request from operation timeout list.");
+        }
+    }
+
+    private void handleTimeoutsAtTime(long currentTime) {
+        Map<Long, InternalRequest> timeouts = requests.subMap(0l, currentTime + 1);
+        for (InternalRequest r : timeouts.values()) {
+            r.handleTimeout(currentTime);
+            requests.values().remove(r);
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java
new file mode 100644
index 00000000000..72155601ba5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java
new file mode 100644
index 00000000000..ab3816d10f3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http.writer;
+
+public class HttpWriter {
+    private final StringBuilder builder = new StringBuilder();
+
+    private String title = "Untitled page";
+    enum State { HEADER, BODY, FINALIZED };
+    private State state = State.HEADER;
+
+    public HttpWriter() {
+    }
+
+    public HttpWriter addTitle(String title) {
+        verifyState(State.HEADER);
+        this.title = title;
+        return this;
+    }
+
+    public HttpWriter write(String paragraph) {
+        verifyState(State.BODY);
+        builder.append("    <p>\n")
+               .append("      " + paragraph + "\n")
+               .append("    </p>\n");
+        return this;
+    }
+
+    public HttpWriter writeLink(String name, String link) {
+        verifyState(State.BODY);
+        builder.append("    <a href=\"" + link + "\">" + name + "</a>\n");
+        return this;
+    }
+
+    private void verifyState(State state) {
+        if (this.state == state) return;
+        if (state != State.FINALIZED && this.state == State.FINALIZED) {
+            throw new IllegalStateException("HTTP page already finalized");
+        }
+        if (state == State.HEADER && this.state == State.BODY) {
+            throw new IllegalStateException("Have already started to write body. Cannot alter header");
+        }
+        if (this.state == State.HEADER) {
+            builder.append("<html>\n"
+                    + "  <head>\n"
+                    + "    <title>" + title + "</title>\n"
+                    + "  </head>\n"
+                    + "  <body>\n"
+                    + "    <h1>" + title + "</h1>\n");
+            this.state = State.BODY;
+            if (this.state == state) return;
+        }
+            // If we get here we are in state body and want to get finalized
+        builder.append("  </body>\n"
+                     + "</html>\n");
+        this.state = State.FINALIZED;
+    }
+
+    public String toString() {
+        verifyState(State.FINALIZED);
+        return builder.toString();
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java
new file mode 100644
index 00000000000..b380130d252
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Interface to implement for backends that want to have a State Rest API.
+ */
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse;
+
+public interface StateRestAPI {
+
+    UnitResponse getState(UnitStateRequest request) throws StateRestApiException;
+
+    SetResponse setUnitState(SetUnitStateRequest request) throws StateRestApiException;
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java
new file mode 100644
index 00000000000..10b629a29e4
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InternalFailure extends StateRestApiException {
+
+    public InternalFailure(String description) {
+        super("Internal failure. Should not happen: " + description);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java
new file mode 100644
index 00000000000..86f6dc81926
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InvalidContentException extends StateRestApiException {
+
+    public InvalidContentException(String description) {
+        super(description);
+        setHtmlCode(400);
+        setHtmlStatus("Content of HTTP request had invalid data");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java
new file mode 100644
index 00000000000..48700e3f343
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InvalidOptionValueException extends StateRestApiException {
+
+    public InvalidOptionValueException(String option, String value, String description) {
+        super("Option '" + option + "' have invalid value '" + value + "': " + description);
+        setHtmlCode(400);
+        setHtmlStatus("Option '" + option + "' have invalid value '" + value + "'");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java
new file mode 100644
index 00000000000..4a0cb76f278
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+/**
+ * @author hakon
+ */
+public class MissingResourceException extends StateRestApiException {
+    public MissingResourceException(String resource) {
+        super("Missing resource '" + resource + "'");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java
new file mode 100644
index 00000000000..037d82fa0b0
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class MissingUnitException extends StateRestApiException {
+
+    private static String createMessage(String[] path, int level) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("No such resource '");
+        for (int i=0; i<=level; ++i) {
+            if (i != 0) sb.append('/');
+            sb.append(path[i]);
+        }
+        return sb.append("'.").toString();
+    }
+
+    public MissingUnitException(String[] path, int level) {
+        super(createMessage(path, level));
+        setHtmlCode(404);
+        setHtmlStatus(getMessage());
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java
new file mode 100644
index 00000000000..46e964d77ef
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public abstract class NotMasterException extends StateRestApiException {
+
+    public NotMasterException(String description) {
+        super(description);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java
new file mode 100644
index 00000000000..e9ebccbfb66
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+import java.util.Arrays;
+
+public class OperationNotSupportedForUnitException extends StateRestApiException {
+
+    private static String createMessage(String[] path, String description) {
+        return new StringBuilder()
+                .append(Arrays.toString(path)).append(": ").append(description)
+                .toString();
+    }
+
+    public OperationNotSupportedForUnitException(String path[], String description) {
+        super(createMessage(path, description));
+        setHtmlCode(405);
+        setHtmlStatus("Operation not supported for resource");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java
new file mode 100644
index 00000000000..4a9c4fc60db
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class OtherMasterException extends NotMasterException {
+    private final String masterHost;
+    private final int masterPort;
+
+    public OtherMasterException(String masterHost, int masterPort) {
+        super("Cluster controller not master. Use master at " + masterHost + ":" + masterPort + ".");
+        this.masterHost = masterHost;
+        this.masterPort = masterPort;
+    }
+
+    public String getHost() { return masterHost; }
+    public int getPort() { return masterPort; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java
new file mode 100644
index 00000000000..6509b31c2c9
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public abstract class StateRestApiException extends Exception {
+    private Integer htmlCode;
+    private String htmlStatus;
+
+    public StateRestApiException(String description) {
+        super(description);
+    }
+
+    /**
+     * If given, this HTML code is set in the response. If not given, a value will
+     * be autogenerated to fit.
+     */
+    public StateRestApiException setHtmlCode(int code) {
+        htmlCode = code;
+        return this;
+    }
+
+    /**
+     * If given, this HTML status string is set in the response. If not given, a value will
+     * be autogenerated to fit.
+     */
+    public StateRestApiException setHtmlStatus(String status) {
+        htmlStatus = status;
+        return this;
+    }
+
+    public Integer getCode() { return htmlCode; }
+    public String getStatus() { return htmlStatus; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
new file mode 100644
index 00000000000..408d4c05092
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class UnknownMasterException extends NotMasterException {
+
+    public UnknownMasterException() {
+        super("No known master cluster controller currently exists.");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java
new file mode 100644
index 00000000000..6925d804cdb
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java
new file mode 100644
index 00000000000..9111172a601
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
new file mode 100644
index 00000000000..972b0c4b82a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState;
+
+import java.util.Map;
+
+public interface SetUnitStateRequest extends UnitRequest {
+
+    Map<String, UnitState> getNewState();
+
+    enum Condition {
+        FORCE(1), // Don't check for any condition before setting unit state
+        SAFE(2); // Only set condition if it is deemed safe (e.g. redundancy is still ok during upgrade)
+
+        public final int value;
+
+        private Condition(int value) {
+            this.value = value;
+        }
+
+        public static Condition fromString(String value) throws InvalidContentException {
+            try {
+                return Condition.valueOf(value.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                throw new InvalidContentException("Invalid value for my enum Condition: " + value);
+            }
+        }
+    }
+    Condition getCondition();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java
new file mode 100644
index 00000000000..4259e837078
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java
@@ -0,0 +1,6 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+public interface UnitRequest {
+    String[] getUnitPath();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java
new file mode 100644
index 00000000000..a2b0210362f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+public interface UnitStateRequest extends UnitRequest {
+    public int getRecursiveLevels();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java
new file mode 100644
index 00000000000..bbd1884e56f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java
new file mode 100644
index 00000000000..20a1f42794c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface CurrentUnitState {
+    public Map<String, UnitState> getStatePerType();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java
new file mode 100644
index 00000000000..9f4ecac84d4
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+/**
+ * The response of a set operation.
+ * @author dybdahl
+ */
+public class SetResponse {
+    private final String reason;
+    private final boolean wasModified;
+
+    public SetResponse(String reason, boolean wasModified) {
+        this.reason = reason;
+        this.wasModified = wasModified;
+    }
+
+    /**
+     * Indicates if data was modified in a set operation.
+     * @return true if modified.
+     */
+    public boolean getWasModified() { return wasModified; }
+
+    /**
+     * Human readable reason.
+     * @return reason as string
+     */
+    public String getReason() { return reason; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java
new file mode 100644
index 00000000000..9415a0f9953
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface SubUnitList {
+    /** id to link map. */
+    public Map<String, String> getSubUnitLinks();
+    public Map<String, UnitResponse> getSubUnits();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java
new file mode 100644
index 00000000000..9c5cb6940b5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitAttributes {
+    public Map<String, String> getAttributeValues();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java
new file mode 100644
index 00000000000..64b8c88584b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitMetrics {
+    public Map<String, Number> getMetricMap();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java
new file mode 100644
index 00000000000..278c3f6dd9b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitResponse {
+    public UnitAttributes getAttributes();
+    public CurrentUnitState getCurrentState();
+    public Map<String, SubUnitList> getSubUnits();
+    public UnitMetrics getMetrics();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java
new file mode 100644
index 00000000000..a4ecf2b09f9
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+public interface UnitState {
+    public String getId();
+    public String getReason();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java
new file mode 100644
index 00000000000..516e48e8047
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
new file mode 100644
index 00000000000..a25f29f005c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class JsonReader {
+    private static class UnitStateImpl implements UnitState {
+        private final String id;
+        private final String reason;
+
+        public UnitStateImpl(String id, String reason) {
+            this.id = id;
+            this.reason = reason;
+        }
+
+        @Override
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public String getReason() {
+            return reason;
+        }
+    }
+
+    static class SetRequestData {
+        final Map<String, UnitState> stateMap;
+        final SetUnitStateRequest.Condition condition;
+        public SetRequestData(Map<String, UnitState> stateMap, SetUnitStateRequest.Condition condition) {
+            this.stateMap = stateMap;
+            this.condition = condition;
+        }
+    }
+
+    public SetRequestData getStateRequestData(HttpRequest request) throws Exception {
+        JSONObject json = new JSONObject(request.getPostContent().toString());
+
+        final SetUnitStateRequest.Condition condition;
+
+        if (json.has("condition")) {
+            condition = SetUnitStateRequest.Condition.valueOf(json.getString("condition"));
+        } else {
+            condition = SetUnitStateRequest.Condition.FORCE;
+        }
+
+        Map<String, UnitState> stateMap = new HashMap<>();
+        if (!json.has("state")) {
+            throw new InvalidContentException("Set state requests must contain a state object");
+        }
+        Object o = json.get("state");
+        if (!(o instanceof JSONObject)) {
+            throw new InvalidContentException("value of state is not a json object");
+        }
+
+        JSONObject state = (JSONObject) o;
+
+        JSONArray stateTypes = state.names();
+        for (int i=0; i<stateTypes.length(); ++i) {
+            o = stateTypes.get(i);
+            String type = (String) o;
+            o = state.get(type);
+            if (!(o instanceof JSONObject)) {
+                throw new InvalidContentException("value of state->" + type + " is not a json object");
+            }
+            JSONObject userState = (JSONObject) o;
+            String code = "up";
+            if (userState.has("state")) {
+                o = userState.get("state");
+                if (!(o instanceof String)) {
+                    throw new InvalidContentException("value of state->" + type + "->state is not a string");
+                }
+                code = o.toString();
+            }
+            String reason = "";
+            if (userState.has("reason")) {
+                o = userState.get("reason");
+                if (!(o instanceof String)) {
+                    throw new InvalidContentException("value of state->" + type + "->reason is not a string");
+                }
+                reason = o.toString();
+            }
+            stateMap.put(type, new UnitStateImpl(code, reason));
+        }
+        return new SetRequestData(stateMap, condition);
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java
new file mode 100644
index 00000000000..f14637c5fb7
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.Map;
+
+public class JsonWriter {
+    private String pathPrefix = "/";
+
+    public JsonWriter() {
+    }
+
+    public void setDefaultPathPrefix(String defaultPathPrefix) {
+        if (defaultPathPrefix.isEmpty() || defaultPathPrefix.charAt(0) != '/') {
+            throw new IllegalArgumentException("Path prefix must start with a slash");
+        }
+        this.pathPrefix = defaultPathPrefix;
+    }
+
+    public JSONObject createJson(UnitResponse data) throws Exception {
+        JSONObject json = new JSONObject();
+        fillInJson(data, json);
+        return json;
+    }
+
+    public void fillInJson(UnitResponse data, JSONObject json) throws Exception {
+        UnitAttributes attributes = data.getAttributes();
+        if (attributes != null) fillInJson(attributes, json);
+        CurrentUnitState stateData = data.getCurrentState();
+        if (stateData != null) fillInJson(stateData, json);
+        UnitMetrics metrics = data.getMetrics();
+        if (metrics != null) fillInJson(metrics, json);
+        Map<String, SubUnitList> subUnits = data.getSubUnits();
+        if (subUnits != null) fillInJson(subUnits, json);
+    }
+
+    public void fillInJson(CurrentUnitState stateData, JSONObject json) throws Exception {
+        JSONObject stateJson = new JSONObject();
+        json.put("state", stateJson);
+        Map<String, UnitState> state = stateData.getStatePerType();
+        for (Map.Entry<String, UnitState> e : state.entrySet()) {
+            String stateType = e.getKey();
+            UnitState unitState = e.getValue();
+            JSONObject stateTypeJson = new JSONObject()
+                    .put("state", unitState.getId())
+                    .put("reason", unitState.getReason());
+            stateJson.put(stateType, stateTypeJson);
+        }
+    }
+
+    public void fillInJson(UnitMetrics metrics, JSONObject json) throws Exception {
+        JSONObject metricsJson = new JSONObject();
+        for (Map.Entry<String, Number> e : metrics.getMetricMap().entrySet()) {
+            metricsJson.put(e.getKey(), e.getValue());
+        }
+        json.put("metrics", metricsJson);
+    }
+    public void fillInJson(UnitAttributes attributes, JSONObject json) throws Exception {
+        JSONObject attributesJson = new JSONObject();
+        for (Map.Entry<String, String> e : attributes.getAttributeValues().entrySet()) {
+            attributesJson.put(e.getKey(), e.getValue());
+        }
+        json.put("attributes", attributesJson);
+    }
+
+    public void fillInJson(Map<String, SubUnitList> subUnitMap, JSONObject json) throws Exception {
+        for(Map.Entry<String, SubUnitList> e : subUnitMap.entrySet()) {
+            String subUnitType = e.getKey();
+            JSONObject typeJson = new JSONObject();
+            for (Map.Entry<String, String> f : e.getValue().getSubUnitLinks().entrySet()) {
+                JSONObject linkJson = new JSONObject();
+                linkJson.put("link", pathPrefix + "/" + f.getValue());
+                typeJson.put(f.getKey(), linkJson);
+            }
+            for (Map.Entry<String, UnitResponse> f : e.getValue().getSubUnits().entrySet()) {
+                JSONObject subJson = new JSONObject();
+                fillInJson(f.getValue(), subJson);
+                typeJson.put(f.getKey(), subJson);
+            }
+            json.put(subUnitType, typeJson);
+        }
+    }
+
+    public JSONObject createErrorJson(String description) {
+        JSONObject o = new JSONObject();
+        try{
+            o.put("message", description);
+        } catch (JSONException e) {
+            // Can't really do anything if we get an error trying to report an error.
+        }
+        return o;
+    }
+
+    public JSONObject createJson(SetResponse setResponse) throws JSONException {
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("wasModified", setResponse.getWasModified());
+        jsonObject.put("reason", setResponse.getReason());
+        return jsonObject;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
new file mode 100644
index 00000000000..43208237bbe
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
@@ -0,0 +1,166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.JsonHttpResult;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.StateRestAPI;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.*;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class RestApiHandler implements HttpRequestHandler {
+    private final static Logger log = Logger.getLogger(RestApiHandler.class.getName());
+
+    private final StateRestAPI restApi;
+    private final JsonWriter jsonWriter;
+    private final JsonReader jsonReader = new JsonReader();
+
+    public RestApiHandler(StateRestAPI restApi) {
+        this.restApi = restApi;
+        this.jsonWriter = new JsonWriter();
+    }
+
+    public RestApiHandler setDefaultPathPrefix(String defaultPathPrefix) {
+        jsonWriter.setDefaultPathPrefix(defaultPathPrefix);
+        return this;
+    }
+
+    private static void logRequestException(HttpRequest request, Exception exception, Level level) {
+        String exceptionString = Exceptions.toMessageString(exception);
+        log.log(level, "Failed to process request with URI path " +
+                request.getPath() + ": " + exceptionString);
+    }
+
+    @Override
+    public HttpResult handleRequest(HttpRequest request) {
+        try{
+            final String[] unitPath = createUnitPath(request);
+            if (request.getHttpOperation().equals(HttpRequest.HttpOp.GET)) {
+                final int recursiveLevel = getRecursiveLevel(request);
+                UnitResponse data = restApi.getState(new UnitStateRequest() {
+                    @Override
+                    public int getRecursiveLevels() {
+                        return recursiveLevel;
+                    }
+                    @Override
+                    public String[] getUnitPath() {
+                        return unitPath;
+                    }
+                });
+                return new JsonHttpResult().setJson(jsonWriter.createJson(data));
+            } else {
+                final JsonReader.SetRequestData setRequestdata = jsonReader.getStateRequestData(request);
+                SetResponse setResponse = restApi.setUnitState(new SetUnitStateRequest() {
+                    @Override
+                    public Map<String, UnitState> getNewState() {
+                        return setRequestdata.stateMap;
+                    }
+                    @Override
+                    public String[] getUnitPath() {
+                        return unitPath;
+                    }
+                    @Override
+                    public Condition getCondition() { return setRequestdata.condition; }
+                });
+                return new JsonHttpResult().setJson(jsonWriter.createJson(setResponse));
+            }
+        } catch (OtherMasterException exception) {
+            logRequestException(request, exception, LogLevel.DEBUG);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(307, "Temporary Redirect");
+            result.addHeader("Location", getMasterLocationUrl(request, exception.getHost(), exception.getPort()));
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (UnknownMasterException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(503, "Service Unavailable");
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (StateRestApiException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(500, "Failed to process request");
+            if (exception.getStatus() != null) result.setHttpCode(result.getHttpReturnCode(), exception.getStatus());
+            if (exception.getCode() != null) result.setHttpCode(exception.getCode(), result.getHttpReturnCodeDescription());
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (Exception exception) {
+            logRequestException(request, exception, LogLevel.ERROR);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(500, "Failed to process request");
+            result.setJson(jsonWriter.createErrorJson(exception.getClass().getName() + ": " + exception.getMessage()));
+            return result;
+        }
+    }
+
+    private String[] createUnitPath(HttpRequest request) {
+        List<String> path = Arrays.asList(request.getPath().split("/"));
+        return path.subList(3, path.size()).toArray(new String[0]);
+    }
+
+    private int getRecursiveLevel(HttpRequest request) throws StateRestApiException {
+        String val = request.getOption("recursive", "false");
+        if (val.toLowerCase().equals("false")) { return 0; }
+        if (val.toLowerCase().equals("true")) { return Integer.MAX_VALUE; }
+        int level;
+        try{
+            level = Integer.parseInt(val);
+            if (level < 0) throw new NumberFormatException();
+        } catch (NumberFormatException e) {
+            throw new InvalidOptionValueException(
+                    "recursive", val, "Recursive option must be true, false, 0 or a positive integer");
+        }
+        return level;
+    }
+
+    private String getMasterLocationUrl(HttpRequest request, String host, int port) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("http://").append(host).append(':').append(port)
+          .append(request.getPath());
+        if (!request.getUrlOptions().isEmpty()) {
+            boolean first = true;
+            for (HttpRequest.KeyValuePair kvp : request.getUrlOptions()) {
+                sb.append(first ? '?' : '&');
+                first = false;
+                sb.append(httpEscape(kvp.getKey())).append('=').append(httpEscape(kvp.getValue()));
+            }
+        }
+        return sb.toString();
+    }
+
+    private static class Escape {
+        public final String pattern;
+        public final String replaceWith;
+
+        public Escape(String pat, String repl) {
+            this.pattern = pat;
+            this.replaceWith = repl;
+        }
+    }
+    private static List<Escape> escapes = new ArrayList<>();
+    static {
+        escapes.add(new Escape("%", "%25"));
+        escapes.add(new Escape(" ", "%20"));
+        escapes.add(new Escape("\\?", "%3F"));
+        escapes.add(new Escape("=", "%3D"));
+        escapes.add(new Escape("\\&", "%26"));
+    }
+
+    private static String httpEscape(String value) {
+        for(Escape e : escapes) {
+            value = value.replaceAll(e.pattern, e.replaceWith);
+        }
+        return value;
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java
new file mode 100644
index 00000000000..f14520b441c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java
new file mode 100644
index 00000000000..f143c3930bf
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import java.util.logging.Logger;
+
+/**
+ * Unit tests want to fast forward time to avoid waiting for time to pass
+ */
+public class FakeClock extends SettableClock {
+    private static final Logger logger = Logger.getLogger(FakeClock.class.getName());
+    protected long currentTime = 1;
+
+    @Override
+    public long getTimeInMillis() {
+        return currentTime;
+    }
+
+    @Override
+    public void adjust(long adjustment) {
+        synchronized (this) {
+            logger.fine("Adjusting clock, adding " + adjustment + " ms to it.");
+            currentTime += adjustment;
+            notifyAll();
+        }
+    }
+
+    @Override
+    public void set(long newTime) {
+        synchronized (this) {
+            if (newTime < currentTime) {
+                // throw new IllegalArgumentException("Clock attempted to be set to go backwards");
+            }
+            currentTime = newTime;
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java
new file mode 100644
index 00000000000..09ae6d3d510
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import com.yahoo.vespa.clustercontroller.utils.util.Clock;
+
+public abstract class SettableClock extends Clock {
+    public abstract void set(long newTime);
+    public abstract void adjust(long adjustment);
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java
new file mode 100644
index 00000000000..032ec5bfe12
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.AsyncHttpClient;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+/**
+ * This class is a utility for unit tests.. You can register HttpRequestHandler instances in it, and then
+ * you can extract an AsyncHttpClient&lt;HttpResult&gt; instance from it, which you can use to talk to the
+ * registered servers. Thus you can do end to end testing of components talking over HTTP without actually
+ * going through HTTP if you are using the HTTP abstraction layer in communication.http package.
+ */
+public class TestTransport {
+    private static final Logger log = Logger.getLogger(TestTransport.class.getName());
+    private static class Handler {
+        HttpRequestHandler handler;
+        String pathPrefix;
+        Handler(HttpRequestHandler h, String prefix) { this.handler = h; this.pathPrefix = prefix; }
+    }
+    private static class Socket {
+        public final String hostname;
+        public final int port;
+
+        Socket(String hostname, int port) {
+            this.hostname = hostname;
+            this.port = port;
+        }
+        @Override
+        public boolean equals(Object o) {
+            if (!(o instanceof Socket)) return false;
+            Socket other = (Socket) o;
+            return (hostname.equals(other.hostname) && port == other.port);
+        }
+        @Override
+        public int hashCode() {
+            return hostname.hashCode() * port;
+        }
+    }
+    private static class Request {
+        public final HttpRequest request;
+        public final AsyncOperationImpl<HttpResult> result;
+
+        Request(HttpRequest r, AsyncOperationImpl<HttpResult> rr) {
+            this.request = r;
+            this.result = rr;
+        }
+    }
+    private final Map<Socket, List<Handler>> handlers = new HashMap<>();
+    private final LinkedList<Request> requests = new LinkedList<>();
+    private final AsyncHttpClient<HttpResult> client = new AsyncHttpClient<HttpResult>() {
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            log.fine("Queueing request " + r);
+            if (r.getHttpOperation() == null) {
+                r = r.clone();
+                r.setHttpOperation(r.getPostContent() == null ? HttpRequest.HttpOp.GET : HttpRequest.HttpOp.POST);
+            }
+            r.verifyComplete();
+            AsyncOperationImpl<HttpResult> op = new AsyncOperationImpl<>(r.toString());
+            synchronized (requests) {
+                requests.addLast(new Request(r, op));
+            }
+            return op;
+        }
+        @Override
+        public void close() { TestTransport.this.close(); }
+    };
+    private boolean running = true;
+    private final Thread workerThread = new Thread() {
+        @Override
+        public void run() {
+            while (running) {
+                synchronized (requests) {
+                    if (requests.isEmpty()) {
+                        try {
+                            requests.wait(100);
+                        } catch (InterruptedException e) { return; }
+                    } else {
+                        Request request = requests.removeFirst();
+                        HttpRequest r = request.request;
+                        log.fine("Processing request " + r);
+                        HttpRequestHandler handler = getHandler(r);
+                        if (handler == null) {
+                            if (log.isLoggable(Level.FINE)) {
+                                log.fine("Failed to find target for request " + r.toString(true));
+                                log.fine("Existing handlers:");
+                                for (Socket socket : handlers.keySet()) {
+                                    log.fine("  Socket " + socket.hostname + ":" + socket.port);
+                                    for (Handler h : handlers.get(socket)) {
+                                        log.fine("    " + h.pathPrefix);
+                                    }
+                                }
+                            }
+                            request.result.setResult(new HttpResult().setHttpCode(
+                                    404, "No such server socket with suitable path prefix found open"));
+                        } else {
+                            try{
+                                request.result.setResult(handler.handleRequest(r));
+                            } catch (Exception e) {
+                                HttpResult result = new HttpResult().setHttpCode(500, e.getMessage());
+                                StringWriter sw = new StringWriter();
+                                e.printStackTrace(new PrintWriter(sw));
+                                result.setContent(sw.toString());
+                                request.result.setResult(result);
+                            }
+                        }
+                        //log.fine("Request " + r.toString(true) + " created result " + request.getSecond().getResult().toString(true));
+                    }
+                }
+            }
+        }
+    };
+
+    public TestTransport() {
+        workerThread.start();
+    }
+
+    public void close() {
+        if (!running) return;
+        running = false;
+        synchronized (requests) { requests.notifyAll(); }
+        try {
+            workerThread.join();
+        } catch (InterruptedException e) {}
+    }
+
+    /** Get an HTTP client that talks to this test transport layer. */
+    public AsyncHttpClient<HttpResult> getClient() { return client; }
+
+    private HttpRequestHandler getHandler(HttpRequest r) {
+        Socket socket = new Socket(r.getHost(), r.getPort());
+        synchronized (this) {
+            List<Handler> handlerList = handlers.get(socket);
+            if (handlerList == null) {
+                log.fine("No socket match");
+                return null;
+            }
+            log.fine("Socket found");
+            for (Handler h : handlers.get(socket)) {
+                if (r.getPath().length() >= h.pathPrefix.length() && r.getPath().substring(0, h.pathPrefix.length()).equals(h.pathPrefix)) {
+                    return h.handler;
+                }
+            }
+            log.fine("No path prefix match");
+        }
+        return null;
+    }
+
+    public void addServer(HttpRequestHandler server, String hostname, int port, String pathPrefix) {
+        Socket socket = new Socket(hostname, port);
+        synchronized (this) {
+            List<Handler> shandlers = handlers.get(socket);
+            if (shandlers == null) {
+                shandlers = new LinkedList<>();
+                handlers.put(socket, shandlers);
+            }
+            shandlers.add(new Handler(server, pathPrefix));
+        }
+    }
+
+    public void removeServer(HttpRequestHandler server, String hostname, int port, String pathPrefix) {
+        Socket socket = new Socket(hostname, port);
+        synchronized (this) {
+            List<Handler> shandlers = handlers.get(socket);
+            if (shandlers == null) return;
+            for (Handler h : shandlers) {
+                if (h.handler == server && h.pathPrefix.equals(pathPrefix)) {
+                    shandlers.remove(h);
+                }
+            }
+        }
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java
new file mode 100644
index 00000000000..d8dccfdd836
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * To avoid having to catch clone not supported exception everywhere, and create code with lack of
+ * coverage, this class exist to hide the clone not supported exceptions that should never happen.
+ */
+public class CertainlyCloneable<T> implements Cloneable {
+    @Override
+    public Object clone() {
+        try{
+            return callParentClone();
+        } catch (CloneNotSupportedException e) {
+                // Super clone should never throw exception for objects that should certainly be cloneable.
+            throw new Error(e);
+        }
+    }
+
+    protected Object callParentClone() throws CloneNotSupportedException {
+        return super.clone();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java
new file mode 100644
index 00000000000..fae3983e2d3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * Wrap access to clock so that we can override it in unit tests
+ */
+public class Clock {
+    public long getTimeInMillis() { return System.currentTimeMillis(); }
+
+    public int getTimeInSecs() { return (int)(getTimeInMillis() / 1000); }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java
new file mode 100644
index 00000000000..d047dcb6bbb
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Metric reporter wrapper to add component name prefix and common dimensions.
+ */
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class ComponentMetricReporter implements MetricReporter {
+    private final MetricReporter impl;
+    private final String prefix;
+    private final Map<String, String> defaultDimensions = new TreeMap<>();
+    private Context defaultContext;
+
+    public ComponentMetricReporter(MetricReporter impl, String prefix) {
+        this.impl = impl;
+        this.prefix = prefix;
+        defaultContext = impl.createContext(defaultDimensions);
+    }
+
+    public ComponentMetricReporter addDimension(String key, String value) {
+        defaultDimensions.put(key, value);
+        defaultContext = impl.createContext(defaultDimensions);
+        return this;
+    }
+
+    public void set(String name, Number value) {
+        impl.set(prefix + name, value, defaultContext);
+    }
+
+    public void add(String name, Number value) {
+        impl.add(prefix + name, value, defaultContext);
+    }
+
+    @Override
+    public void set(String name, Number value, Context context) {
+        impl.set(prefix + name, value, context);
+    }
+
+    @Override
+    public void add(String name, Number value, Context context) {
+        impl.add(prefix + name, value, context);
+    }
+
+    @Override
+    public Context createContext(Map<String, ?> stringMap) {
+        if (stringMap == null) return defaultContext;
+        Map<String, Object> m = new TreeMap<>(stringMap);
+        for(String key : defaultDimensions.keySet()) {
+            if (!m.containsKey(key)) m.put(key, defaultDimensions.get(key));
+        }
+        return impl.createContext(m);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java
new file mode 100644
index 00000000000..ca9561ac85e
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * The Jettison json object class has an interface issue where it hides null pointer exceptions
+ * as checked json exceptions. Consequently one has to create catch clauses that code cannot get
+ * into. This class hides those exceptions.
+ *
+ * (Add functions to this wrapper for new functions needing to hide exceptions like this as they are
+ * needed)
+ */
+public class JSONObjectWrapper extends JSONObject {
+
+    @Override
+    public JSONObjectWrapper put(String key, Object value) {
+        try{
+            super.put(key, value);
+            return this;
+        } catch (JSONException e) {
+            throw new NullPointerException(e.getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java
new file mode 100644
index 00000000000..839eebf4bda
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * Wrapper for current jdisc metrics, such that applications can report metrics without depending on
+ * the whole container. The apputil project will provide an implementation of this interface that
+ * reports on to injected jdisc implementation.
+ */
+public interface MetricReporter {
+    void set(java.lang.String s, java.lang.Number number, Context context);
+
+    void add(java.lang.String s, java.lang.Number number, Context context);
+
+    Context createContext(java.util.Map<java.lang.String,?> dimensions);
+    static interface Context {
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java
new file mode 100644
index 00000000000..ffe6027ce4a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import java.util.Map;
+
+public class NoMetricReporter implements MetricReporter {
+    @Override
+    public void set(String s, Number number, Context context) {}
+
+    @Override
+    public void add(String s, Number number, Context context) {}
+
+    @Override
+    public Context createContext(Map<String, ?> stringMap) { return null; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java
new file mode 100644
index 00000000000..92aae9f4778
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java
new file mode 100644
index 00000000000..855dc2c7263
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java
@@ -0,0 +1,285 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import junit.framework.TestCase;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.LinkedList;
+
+public class AsyncTest extends TestCase {
+
+    public void testListeners() {
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        class Listener implements AsyncCallback<String> {
+            boolean called = false;
+            @Override
+            public void done(AsyncOperation<String> op) {
+                called = true;
+            }
+        }
+        Listener l1 = new Listener();
+        Listener l2 = new Listener();
+        Listener l3 = new Listener();
+        Listener l4 = new Listener();
+        op.register(l1);
+        op.register(l2);
+        op.register(l3);
+        op.unregister(l1);
+        op.setResult("foo");
+        op.register(l4);
+            // Listener that is unregistered is not called
+        assertEquals(false, l1.called);
+            // Listener that is registered is called
+        assertEquals(true, l2.called);
+            // Multiple listeners supported
+        assertEquals(true, l3.called);
+            // Listener called directly when registered after result is set
+        assertEquals(true, l4.called);
+    }
+
+    public void testMultipleResultSetters() {
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setResult("foo");
+            op.setResult("bar");
+            assertEquals("foo", op.getResult());
+        }
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setResult("foo");
+            op.setFailure(new Exception("bar"));
+            assertEquals("foo", op.getResult());
+            assertEquals(true, op.isSuccess());
+        }
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setFailure(new Exception("bar"));
+            op.setResult("foo");
+            assertNull(op.getResult());
+            assertEquals(false, op.isSuccess());
+            assertEquals("bar", op.getCause().getMessage());
+        }
+    }
+
+    public void testPartialResultOnFailure() {
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        op.setFailure(new Exception("bar"), "foo");
+        assertEquals("foo", op.getResult());
+        assertEquals(false, op.isSuccess());
+        assertEquals("bar", op.getCause().getMessage());
+    }
+
+    public void testListenImpl() {
+        class ListenImpl extends AsyncOperationListenImpl<String> {
+            public ListenImpl(AsyncOperation<String> op) {
+                super(op);
+            }
+        };
+        class Listener implements AsyncCallback<String> {
+            int calls = 0;
+            @Override
+            public void done(AsyncOperation<String> op) {
+                ++calls;
+            }
+        }
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        ListenImpl impl = new ListenImpl(op);
+        Listener l1 = new Listener();
+        impl.register(l1);
+        impl.notifyListeners();
+        impl.notifyListeners();
+        impl.notifyListeners();
+        assertEquals(1, l1.calls);
+    }
+
+    public void testRedirectedOperation() {
+        {
+            final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test", "desc");
+            AsyncOperation<Integer> deleteRequest = new RedirectedAsyncOperation<String, Integer>(op) {
+                @Override
+                public Integer getResult() {
+                    return Integer.valueOf(op.getResult());
+                }
+            };
+            final LinkedList<Integer> result = new LinkedList<>();
+            deleteRequest.register(new AsyncCallback<Integer>() {
+                @Override
+                public void done(AsyncOperation<Integer> op) {
+                    result.add(op.getResult());
+                }
+            });
+            assertNull(deleteRequest.getProgress());
+            op.setResult("123");
+            assertEquals(true, deleteRequest.isDone());
+            assertEquals(true, deleteRequest.isSuccess());
+            assertEquals(new Integer(123), deleteRequest.getResult());
+            assertEquals("desc", deleteRequest.getDescription());
+            assertEquals("test", deleteRequest.getName());
+            assertEquals(1, result.size());
+            assertEquals(Integer.valueOf(123), result.getFirst());
+            assertEquals(Double.valueOf(1.0), deleteRequest.getProgress());
+
+            // Get some extra coverage
+            deleteRequest.cancel();
+            deleteRequest.isCanceled();
+            deleteRequest.unregister(null);
+        }
+        {
+            final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test", "desc");
+            AsyncOperation<Integer> deleteRequest = new RedirectedAsyncOperation<String, Integer>(op) {
+                @Override
+                public Integer getResult() {
+                    return Integer.valueOf(op.getResult());
+                }
+            };
+            op.setFailure(new Exception("foo"));
+            assertEquals(true, deleteRequest.isDone());
+            assertEquals("foo", deleteRequest.getCause().getMessage());
+            assertEquals(false, deleteRequest.isSuccess());
+            deleteRequest.getProgress();
+        }
+    }
+
+    public void testRedirectOnSuccessOperation() {
+        {
+            final AsyncOperationImpl<Integer> target = new AsyncOperationImpl<>("foo");
+            SuccessfulAsyncCallback<String, Integer> callback = new SuccessfulAsyncCallback<String, Integer>(target) {
+                @Override
+                public void successfullyDone(AsyncOperation<String> source) {
+                    target.setResult(Integer.valueOf(source.getResult()));
+                }
+            };
+            AsyncOperationImpl<String> source = new AsyncOperationImpl<>("source");
+            source.register(callback);
+            source.setResult("5");
+            assertTrue(target.isDone());
+            assertTrue(target.isSuccess());
+            assertEquals(new Integer(5), target.getResult());
+        }
+        {
+            final AsyncOperationImpl<Integer> target = new AsyncOperationImpl<>("foo");
+            SuccessfulAsyncCallback<String, Integer> callback = new SuccessfulAsyncCallback<String, Integer>(target) {
+                @Override
+                public void successfullyDone(AsyncOperation<String> source) {
+                    target.setResult(Integer.valueOf(source.getResult()));
+                }
+            };
+            AsyncOperationImpl<String> source = new AsyncOperationImpl<>("source");
+            source.register(callback);
+            source.setFailure(new RuntimeException("foo"));
+            assertTrue(target.isDone());
+            assertFalse(target.isSuccess());
+            assertEquals("foo", target.getCause().getMessage());
+        }
+    }
+
+    private abstract class StressThread implements Runnable {
+        private final Object monitor;
+        private boolean running = true;
+
+        public StressThread(Object monitor) { this.monitor = monitor; }
+
+        public void stop() {
+            synchronized (monitor) {
+                running = false;
+                monitor.notifyAll();
+            }
+        }
+
+        @Override
+        public void run() {
+            try{ synchronized (monitor) { while (running) {
+                if (hasTask()) {
+                    doTask();
+                } else {
+                    monitor.wait(1000);
+                }
+            } } } catch (Exception e) {}
+        }
+
+        public abstract boolean hasTask();
+        public abstract void doTask();
+    }
+
+    private abstract class AsyncOpStressThread extends StressThread {
+        public AsyncOperationImpl<String> op;
+        public AsyncOpStressThread(Object monitor) { super(monitor); }
+        @Override
+        public boolean hasTask() { return op != null; }
+    }
+
+    private class Completer extends AsyncOpStressThread {
+        public Completer(Object monitor) { super(monitor); }
+        @Override
+        public void doTask() { op.setResult("foo"); op = null; }
+    }
+
+    private class Listener extends AsyncOpStressThread implements AsyncCallback<String> {
+        int counter = 0;
+        int unset = 0;
+        int priorReg = 0;
+        public Listener(Object monitor) { super(monitor); }
+        @Override
+        public void done(AsyncOperation<String> op) {
+            synchronized (this) {
+                if (op.getResult() == null) ++unset;
+                ++counter;
+            }
+        }
+
+        @Override
+        public void doTask() {
+            op.register(this);
+            if (!op.isDone()) ++priorReg;
+            op = null;
+        }
+    }
+
+    public void testStressCompletionAndRegisterToDetectRace() throws Exception {
+        int iterations = 1000;
+        Object monitor = new Object();
+        Completer completer = new Completer(monitor);
+        Listener listener = new Listener(monitor);
+        Thread t1 = new Thread(completer);
+        Thread t2 = new Thread(listener);
+        try{
+            t1.start();
+            t2.start();
+            for (int i=0; i<iterations; ++i) {
+                final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+                synchronized (monitor) {
+                    completer.op = op;
+                    listener.op = op;
+                    monitor.notifyAll();
+                }
+                while (completer.op != null || listener.op != null) {
+                    try{ Thread.sleep(0); } catch (InterruptedException e) {}
+                }
+            }
+        } finally {
+            completer.stop();
+            listener.stop();
+            t1.join();
+            t2.join();
+        }
+        /*
+        System.out.println("Done with " + iterations + " iterations. "
+                        + "Registered prior " + listener.priorReg + " times. "
+                        + "Unset " + listener.unset + " times. ");
+        // */
+        assertEquals(0, listener.unset);
+        assertEquals(iterations, listener.counter);
+    }
+
+    public void ignoreTestExceptionOnCallback() throws Exception {
+        AsyncOperationImpl<String> impl = new AsyncOperationImpl<>("foo");
+        impl.register(new AsyncCallback<String>() {
+            @Override
+            public void done(AsyncOperation<String> op) {
+                throw new RuntimeException("Foo");
+            }
+        });
+        impl.setResult(null);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java
new file mode 100644
index 00000000000..d8e3983cd32
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+public class AsyncHttpClientWithBaseTest extends TestCase {
+
+    public void testOverride() {
+        class HttpClient implements AsyncHttpClient<HttpResult> {
+            HttpRequest lastRequest;
+            @Override
+            public AsyncOperation<HttpResult> execute(HttpRequest r) {
+                lastRequest = r;
+                return new AsyncOperationImpl<>("test");
+            }
+            @Override
+            public void close() {
+            }
+        }
+
+        HttpClient client = new HttpClient();
+        AsyncHttpClientWithBase<HttpResult> base = new AsyncHttpClientWithBase<>(client);
+            // No override by default
+        HttpRequest r = new HttpRequest().setPath("/foo").setHost("bar").setPort(50);
+        base.execute(r);
+        assertEquals(client.lastRequest, r);
+            // Base request always set
+        base.setHttpRequestBase(null);
+        base.execute(r);
+        assertEquals(client.lastRequest, r);
+            // Set an override
+        base.setHttpRequestBase(new HttpRequest().setHttpOperation(HttpRequest.HttpOp.DELETE));
+        base.execute(r);
+        assertNotSame(client.lastRequest, r);
+        assertEquals(HttpRequest.HttpOp.DELETE, client.lastRequest.getHttpOperation());
+
+        base.close();
+    }
+
+    public void testClientMustBeSet() {
+        try{
+            new AsyncHttpClientWithBase<HttpResult>(null);
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java
new file mode 100644
index 00000000000..4ef0b4daccc
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+
+public class DummyAsyncHttpClient implements AsyncHttpClient<HttpResult> {
+    HttpResult result;
+    HttpRequest lastRequest;
+
+    public DummyAsyncHttpClient(HttpResult result) {
+        this.result = result;
+    }
+
+    @Override
+    public AsyncOperation<HttpResult> execute(HttpRequest r) {
+        lastRequest = r;
+        AsyncOperationImpl<HttpResult> op = new AsyncOperationImpl<>(r.toString());
+        op.setResult(result);
+        return op;
+    }
+
+    @Override
+    public void close() {
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java
new file mode 100644
index 00000000000..8bd9cfe5dbe
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+
+public class HttpRequestTest extends TestCase {
+
+    private HttpRequest createRequest() {
+        return new HttpRequest()
+                .setHost("local")
+                .setPort(20)
+                .addHttpHeader("x-foo", "bar")
+                .setPath("/bah")
+                .setHttpOperation(HttpRequest.HttpOp.PUT)
+                .addUrlOption("urk", "arg")
+                .setTimeout(25);
+    }
+
+    public void testEquality() {
+        assertEquals(createRequest(), createRequest());
+        assertNotSame(createRequest(), createRequest().setHost("localhost"));
+        assertNotSame(createRequest(), createRequest().setPort(40));
+        assertNotSame(createRequest(), createRequest().setPath("/hmm"));
+        assertNotSame(createRequest(), createRequest().addHttpHeader("dsf", "fs"));
+        assertNotSame(createRequest(), createRequest().setHttpOperation(HttpRequest.HttpOp.DELETE));
+    }
+
+    public void testVerifyComplete() {
+        // To be a complete request, an HTTP request must include:
+        //  - A path
+        //  - The HTTP operation type
+        try{
+            new HttpRequest().setPath("/foo").verifyComplete();
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        try{
+            new HttpRequest().setHttpOperation(HttpRequest.HttpOp.GET).verifyComplete();
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        new HttpRequest().setHttpOperation(HttpRequest.HttpOp.GET).setPath("/bar").verifyComplete();
+    }
+
+    public void testMerge() {
+        {
+            HttpRequest base = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            HttpRequest req = new HttpRequest()
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            HttpRequest merged = base.merge(req);
+
+            HttpRequest expected = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            assertEquals(expected, merged);
+        }
+        {
+            HttpRequest base = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addHttpHeader("x-foo", "bar")
+                    .addUrlOption("hmm", "arg");
+            HttpRequest req = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.PUT)
+                    .setPath("/gohere")
+                    .addHttpHeader("Content-Type", "whatevah")
+                    .addUrlOption("tit", "tat")
+                    .setPostContent("foo");
+            HttpRequest merged = base.merge(req);
+
+            HttpRequest expected = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.PUT)
+                    .setPath("/gohere")
+                    .addHttpHeader("x-foo", "bar")
+                    .addHttpHeader("Content-Type", "whatevah")
+                    .addUrlOption("hmm", "arg")
+                    .addUrlOption("tit", "tat")
+                    .setPostContent("foo");
+            assertEquals(expected, merged);
+        }
+    }
+
+    public void testNonExistingHeader() {
+        assertEquals("foo", new HttpRequest().getHeader("asd", "foo"));
+        assertEquals("foo", new HttpRequest().addHttpHeader("bar", "foo").getHeader("asd", "foo"));
+    }
+
+    public void testOption() {
+        assertEquals("bar", new HttpRequest().addUrlOption("foo", "bar").getOption("foo", "foo"));
+        assertEquals("foo", new HttpRequest().getOption("asd", "foo"));
+    }
+
+    public void testToString() {
+        assertEquals("GET? http://localhost:8080/",
+                     new HttpRequest()
+                             .setHost("localhost")
+                             .setPort(8080)
+                             .toString(true));
+        assertEquals("POST http://localhost/",
+                new HttpRequest()
+                        .setHttpOperation(HttpRequest.HttpOp.POST)
+                        .setHost("localhost")
+                        .toString(true));
+        assertEquals("GET http://localhost/?foo=bar",
+                new HttpRequest()
+                        .setHttpOperation(HttpRequest.HttpOp.GET)
+                        .addUrlOption("foo", "bar")
+                        .setHost("localhost")
+                        .toString(true));
+    }
+
+    public void testNothingButGetCoverage() {
+        assertEquals(false, new HttpRequest().equals(new Object()));
+        new HttpRequest().getHeaders();
+        new HttpRequest().setUrlOptions(new HttpRequest().getUrlOptions());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java
new file mode 100644
index 00000000000..aa9f019c757
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+
+public class HttpResultTest extends TestCase {
+
+    public void testSuccess() {
+        assertEquals(false, new HttpResult().setHttpCode(199, "foo").isSuccess());
+        assertEquals(true, new HttpResult().setHttpCode(200, "foo").isSuccess());
+        assertEquals(true, new HttpResult().setHttpCode(299, "foo").isSuccess());
+        assertEquals(false, new HttpResult().setHttpCode(300, "foo").isSuccess());
+    }
+
+    public void testToString() {
+        assertEquals("HTTP 200/OK", new HttpResult().setContent("Foo").toString());
+        assertEquals("HTTP 200/OK\n\nFoo", new HttpResult().setContent("Foo").toString(true));
+        assertEquals("HTTP 200/OK", new HttpResult().toString(true));
+        assertEquals("HTTP 200/OK", new HttpResult().setContent("").toString(true));
+    }
+
+    public void testNothingButGetCoverage() {
+        new HttpResult().getHeaders();
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java
new file mode 100644
index 00000000000..b49b72ef463
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonAsyncHttpClientTest extends TestCase {
+
+    public void testJSONInJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+        client.addJsonContentType(true);
+        client.verifyRequestContentAsJson(true);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+        assertTrue(result.isSuccess());
+
+        result.toString();
+        client.close();
+    }
+
+    public void testStringInJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42).toString()));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34).toString());
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+    }
+
+    public void testIllegalJsonIn() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        try {
+            HttpRequest r = new HttpRequest();
+            r.setPostContent("my illegal json");
+
+            client.execute(r);
+            assertTrue(false);
+        } catch (Exception e) {
+
+        }
+    }
+
+    public void testIllegalJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent("my illegal json"));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34).toString());
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals("{\"error\":\"Invalid JSON in output: A JSONObject text must begin with '{' at character 1 of my illegal json\",\"output\":\"my illegal json\"}", result.getResult().getJson().toString());
+    }
+
+    public void testEmptyReply() throws Exception {
+        class Client implements AsyncHttpClient<HttpResult> {
+            AsyncOperationImpl<HttpResult> lastOp;
+            @Override
+            public AsyncOperation<HttpResult> execute(HttpRequest r) {
+                return lastOp = new AsyncOperationImpl<>(r.toString());
+            }
+            @Override
+            public void close() {
+            }
+        };
+        Client client = new Client();
+        JsonAsyncHttpClient jsonClient = new JsonAsyncHttpClient(client);
+        AsyncOperation<JsonHttpResult> op = jsonClient.execute(new HttpRequest());
+        client.lastOp.setResult(null);
+        assertNull(op.getResult());
+    }
+
+    public void testNotVerifyingJson() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+        client.addJsonContentType(true);
+        client.verifyRequestContentAsJson(false);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+        assertTrue(result.isSuccess());
+
+        result.toString();
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java
new file mode 100644
index 00000000000..cfc11e6dc8d
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonHttpResultTest extends TestCase {
+
+    public void testCopyConstructor() {
+        assertEquals("{}", new JsonHttpResult(new HttpResult()).getJson().toString());
+    }
+
+    public void testOutput() {
+        assertEquals("HTTP 200/OK\n"
+                   + "\n"
+                   + "JSON: {\"foo\": 3}",
+                     new JsonHttpResult(new HttpResult().setContent("{ \"foo\" : 3 }")).toString(true));
+        assertEquals("HTTP 200/OK\n"
+                + "\n"
+                + "{ \"foo\" : }",
+                new JsonHttpResult(new HttpResult().setContent("{ \"foo\" : }")).toString(true));
+    }
+
+    public void testNonJsonOutput() {
+        JsonHttpResult result = new JsonHttpResult();
+        result.setContent("Foo");
+        StringBuilder sb = new StringBuilder();
+        result.printContent(sb);
+        assertEquals("Foo", sb.toString());
+    }
+
+    public void testInvalidJsonOutput() {
+        JsonHttpResult result = new JsonHttpResult();
+        result.setJson(new JSONObject() {
+            @Override
+            public String toString(int indent) throws JSONException {
+                throw new JSONException("Foo");
+            }
+        });
+        StringBuilder sb = new StringBuilder();
+        result.printContent(sb);
+        assertEquals("JSON: {}", sb.toString());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java
new file mode 100644
index 00000000000..cb7ac4d5d31
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class LoggingAsyncHttpClientTest extends TestCase {
+    class HttpClient implements AsyncHttpClient<HttpResult> {
+        AsyncOperationImpl<HttpResult> lastOp;
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            return lastOp = new AsyncOperationImpl<>("test");
+        }
+        @Override
+        public void close() {
+        }
+    }
+
+    public void testWithoutDebugLog() throws Exception {
+        doRequests();
+    }
+
+    public void testWithDebugLog() throws Exception {
+        Logger log = Logger.getLogger(LoggingAsyncHttpClient.class.getName());
+        log.setLevel(Level.FINE);
+        doRequests();
+    }
+
+    private void doRequests() {
+        {
+            HttpClient client = new HttpClient();
+            LoggingAsyncHttpClient<HttpResult> loggingClient = new LoggingAsyncHttpClient<>(client);
+            AsyncOperation<HttpResult> op = loggingClient.execute(new HttpRequest());
+            client.lastOp.setResult(new HttpResult().setContent("foo"));
+            assertEquals("foo", op.getResult().getContent());
+        }
+        {
+            HttpClient client = new HttpClient();
+            LoggingAsyncHttpClient<HttpResult> loggingClient = new LoggingAsyncHttpClient<>(client);
+            AsyncOperation<HttpResult> op = loggingClient.execute(new HttpRequest());
+            client.lastOp.setFailure(new Exception("foo"));
+            assertEquals("foo", op.getCause().getMessage());
+        }
+
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java
new file mode 100644
index 00000000000..2c2e7cfcff3
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class ProxyAsyncHttpClientTest extends TestCase {
+
+    public void testSimple() throws Exception {
+        // Can't really test much here, but verifies that the code runs.
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        ProxyAsyncHttpClient client = new ProxyAsyncHttpClient<>(dummy, "myproxyhost", 1234);
+
+        HttpRequest r = new HttpRequest();
+        r.setPath("/foo");
+        r.setHost("myhost");
+        r.setPort(4567);
+
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        client.execute(r);
+
+        assertEquals(new HttpRequest().setPath("/myhost:4567/foo")
+                                      .setHost("myproxyhost")
+                                      .setPort(1234)
+                                      .setPostContent(new JSONObject().put("foo", 34)),
+                     dummy.lastRequest);
+    }
+
+    public void testNoAndEmptyPath() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        ProxyAsyncHttpClient client = new ProxyAsyncHttpClient<>(dummy, "myproxyhost", 1234);
+        try{
+            client.execute(new HttpRequest());
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+            assertTrue(e.getMessage().contains("Host and path must be set prior"));
+        }
+        client.execute(new HttpRequest().setHost("local").setPath(""));
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java
new file mode 100644
index 00000000000..27632047c19
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+import java.util.LinkedList;
+
+public class RequestQueueTest extends TestCase {
+    public static class Request {
+        public final HttpRequest request;
+        public final AsyncOperationImpl<HttpResult> result;
+
+        public Request(HttpRequest r, AsyncOperationImpl<HttpResult> rr) {
+            this.request = r;
+            this.result = rr;
+        }
+    }
+
+    public class TestClient implements AsyncHttpClient<HttpResult> {
+        LinkedList<Request> requests = new LinkedList<>();
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            Request p = new Request(r, new AsyncOperationImpl<HttpResult>(r.toString()));
+            synchronized (requests) {
+                requests.addLast(p);
+            }
+            return p.result;
+        }
+        @Override
+        public void close() {}
+    };
+
+    public void testNormalUsage() {
+        TestClient client = new TestClient();
+        RequestQueue<HttpResult> queue = new RequestQueue<>(client, 4);
+        final LinkedList<HttpResult> results = new LinkedList<>();
+        for (int i=0; i<10; ++i) {
+            queue.schedule(new HttpRequest().setPath("/" + i), new AsyncCallback<HttpResult>() {
+                @Override
+                public void done(AsyncOperation<HttpResult> op) {
+                    if (op.isSuccess()) {
+                        results.add(op.getResult());
+                    } else {
+                        results.add(new HttpResult().setHttpCode(500, op.getCause().getMessage()));
+                    }
+                }
+            });
+        }
+        assertEquals(4, client.requests.size());
+        for (int i=0; i<3; ++i) {
+            Request p = client.requests.removeFirst();
+            p.result.setResult(new HttpResult());
+            assertEquals(true, results.getLast().isSuccess());
+        }
+        assertEquals(4, client.requests.size());
+        for (int i=0; i<7; ++i) {
+            Request p = client.requests.removeFirst();
+            p.result.setFailure(new Exception("Fail"));
+            assertEquals(false, results.getLast().isSuccess());
+        }
+        assertEquals(0, client.requests.size());
+        assertEquals(true, queue.empty());
+        assertEquals(10, results.size());
+    }
+
+    public class Waiter implements Runnable {
+        boolean waiting = false;
+        boolean completed = false;
+        RequestQueue<HttpResult> queue;
+        Waiter(RequestQueue<HttpResult> queue) {
+            this.queue = queue;
+        }
+        public void run() {
+            try{
+                waiting = true;
+                queue.waitUntilEmpty();
+            } catch (InterruptedException e) { throw new Error(e); }
+            completed = true;
+        }
+    }
+
+    public void testWaitUntilEmpty() throws Exception {
+        TestClient client = new TestClient();
+        RequestQueue<HttpResult> queue = new RequestQueue<>(client, 4);
+        final LinkedList<HttpResult> result = new LinkedList<>();
+        queue.schedule(new HttpRequest().setPath("/foo"), new AsyncCallback<HttpResult>() {
+            @Override
+            public void done(AsyncOperation<HttpResult> op) {
+                result.add(op.getResult());
+            }
+        });
+        Waiter waiter = new Waiter(queue);
+        Thread thread = new Thread(waiter);
+        thread.start();
+        while (!waiter.waiting) {
+            Thread.sleep(1);
+        }
+        assertEquals(0, result.size());
+        client.requests.getFirst().result.setResult(new HttpResult());
+        while (!waiter.completed) {
+            Thread.sleep(1);
+        }
+        assertEquals(1, result.size());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java
new file mode 100644
index 00000000000..a52c6b0cfaa
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncUtils;
+import com.yahoo.vespa.clustercontroller.utils.test.FakeClock;
+import junit.framework.TestCase;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class TimeoutHandlerTest extends TestCase {
+
+    public class TestClient implements AsyncHttpClient<HttpResult> {
+        AsyncOperationImpl<HttpResult> lastOp;
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            return lastOp = new AsyncOperationImpl<>("test");
+        }
+        @Override
+        public void close() {}
+    };
+
+    private ThreadPoolExecutor executor;
+    private TestClient client;
+    private FakeClock clock;
+    private TimeoutHandler<HttpResult> handler;
+
+    public void setUp() {
+        executor = new ThreadPoolExecutor(10, 100, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1000));
+        clock = new FakeClock();
+        client = new TestClient();
+        handler = new TimeoutHandler<>(executor, clock, client);
+    }
+
+    public void tearDown() {
+        handler.close();
+        executor.shutdown();
+    }
+
+    public void testTimeout() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        assertFalse(op.isDone());
+        clock.adjust(999);
+            // Give it a bit of time for timeout handler to have a chance of timout out prematurely
+        try{ Thread.sleep(1); } catch (InterruptedException e) {}
+        assertFalse(op.isDone());
+        clock.adjust(1);
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertFalse(op.isSuccess());
+        assertTrue(op.getCause().getMessage(), op.getCause().getMessage().contains("Operation timeout"));
+            // After timeout, finishing the original request no longer matter
+        client.lastOp.setResult(new HttpResult());
+        assertFalse(op.isSuccess());
+        assertTrue(op.getCause().getMessage(), op.getCause().getMessage().contains("Operation timeout"));
+    }
+
+    public void testNoTimeout() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(999);
+        assertFalse(op.isDone());
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertTrue(op.isSuccess());
+        assertEquals("foo", op.getResult().getContent());
+    }
+
+    public void testNoTimeoutFailing() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(999);
+        assertFalse(op.isDone());
+        client.lastOp.setFailure(new Exception("foo"));
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertFalse(op.isSuccess());
+        assertEquals("foo", op.getCause().getMessage());
+    }
+
+    public void testProvokeCompletedOpPurgeInTimeoutList() {
+        AsyncOperation<HttpResult> op1 = handler.execute(new HttpRequest().setTimeout(1000));
+        AsyncOperationImpl<HttpResult> op1Internal = client.lastOp;
+        clock.adjust(300);
+        AsyncOperation<HttpResult> op2 = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(300);
+        op1Internal.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op1);
+        clock.adjust(800);
+        AsyncUtils.waitFor(op2);
+        assertEquals(true, op1.isDone());
+        assertEquals(true, op2.isDone());
+        assertEquals(true, op1.isSuccess());
+        assertEquals(false, op2.isSuccess());
+    }
+
+    public void testNothingButGetCoverage() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        op.getProgress();
+        op.cancel();
+        assertFalse(op.isCanceled()); // Cancel not currently supported
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+        op.getProgress();
+        op = handler.execute(new HttpRequest().setTimeout(1000));
+        handler.performTimeoutHandlerTick();
+        handler.performTimeoutHandlerTick();
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java
new file mode 100644
index 00000000000..e8c47554cc2
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http.writer;
+
+import junit.framework.TestCase;
+
+public class HttpWriterTest extends TestCase {
+    private static String defaultTitle = "My Title";
+    private static String defaultHeader = "<html>\n"
+                                        + "  <head>\n"
+                                        + "    <title>My Title</title>\n"
+                                        + "  </head>\n"
+                                        + "  <body>\n"
+                                        + "    <h1>My Title</h1>\n";
+    private static String defaultFooter = "  </body>\n"
+                                        + "</html>\n";
+
+
+    public void testStructure() {
+        HttpWriter writer = new HttpWriter();
+        String header = defaultHeader.replace(defaultTitle, "Untitled page");
+        assertEquals(header + defaultFooter, writer.toString());
+    }
+    public void testTitle() {
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle);
+        assertEquals(defaultHeader + defaultFooter, writer.toString());
+    }
+    public void testParagraph() {
+        String paragraph = "This is a paragraph";
+        String paragraph2 = "More text";
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle).write(paragraph).write(paragraph2);
+        String content = "    <p>\n"
+                       + "      " + paragraph + "\n"
+                       + "    </p>\n"
+                       + "    <p>\n"
+                       + "      " + paragraph2 + "\n"
+                       + "    </p>\n";
+        assertEquals(defaultHeader + content + defaultFooter, writer.toString());
+    }
+    public void testLink() {
+        String name = "My link";
+        String link = "/foo/bar?hmm";
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle).writeLink(name, link);
+        String content = "    <a href=\"" + link + "\">" + name + "</a>\n";
+        assertEquals(defaultHeader + content + defaultFooter, writer.toString());
+    }
+    public void testErrors() {
+        try{
+            HttpWriter writer = new HttpWriter().addTitle(defaultTitle);
+            writer.toString();
+            writer.write("foo");
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        try{
+            new HttpWriter().write("foo").addTitle("bar");
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java
new file mode 100644
index 00000000000..4f1be08267c
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DummyBackend {
+    public static class Cluster {
+        public String id;
+        public Map<String, Node> nodes = new LinkedHashMap<>();
+
+        public Cluster(String id) { this.id = id; }
+        public Cluster addNode(Node n) { nodes.put(n.id, n); n.clusterId = id; return this; }
+    }
+    public static class Node {
+        public String clusterId;
+        public String id;
+        public int docCount = 0;
+        public String state = "up";
+        public String reason = "";
+        public String group = "mygroup";
+
+        public Node(String id) { this.id = id; }
+
+        public Node setDocCount(int count) { docCount = count; return this; }
+        public Node setState(String state) { this.state = state; return this; }
+    }
+    private Map<String, Cluster> clusters = new LinkedHashMap<>();
+
+    public Map<String, Cluster> getClusters() { return clusters; }
+
+    public DummyBackend addCluster(Cluster c) {
+        clusters.put(c.id, c);
+        return this;
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java
new file mode 100644
index 00000000000..3a5ea520f4a
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java
@@ -0,0 +1,194 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OperationNotSupportedForUnitException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DummyStateApi implements StateRestAPI {
+    private final DummyBackend backend;
+    private Exception induceException;
+
+    public DummyStateApi(DummyBackend backend) {
+        this.backend = backend;
+    }
+
+    public void induceException(StateRestApiException e) {
+        induceException = e;
+    }
+    public void induceException(RuntimeException e) {
+        induceException = e;
+    }
+
+    public class SubUnitListImpl implements SubUnitList {
+        private Map<String, String> links = new LinkedHashMap<>();
+        private Map<String, UnitResponse> values = new LinkedHashMap<>();
+
+        @Override
+        public Map<String, String> getSubUnitLinks() { return links; }
+        @Override
+        public Map<String, UnitResponse> getSubUnits() { return values; }
+
+        public void addUnit(DummyBackend.Cluster cluster, int recursive) {
+            if (recursive == 0) {
+                links.put(cluster.id, cluster.id);
+            } else {
+                values.put(cluster.id, getClusterState(cluster, recursive - 1));
+            }
+        }
+        public void addUnit(DummyBackend.Node node, int recursive) {
+            if (recursive == 0) {
+                String link = node.clusterId + '/' + node.id;
+                links.put(node.id, link);
+            } else {
+                values.put(node.id, getNodeState(node));
+            }
+        }
+    }
+
+    private UnitResponse getClusterList(final int recursive) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() { return null; }
+            @Override
+            public UnitMetrics getMetrics() { return null; }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() {
+                Map<String, SubUnitList> result = new LinkedHashMap<>();
+                SubUnitListImpl subUnits = new SubUnitListImpl();
+                result.put("cluster", subUnits);
+                for (Map.Entry<String, DummyBackend.Cluster> e : backend.getClusters().entrySet()) {
+                    subUnits.addUnit(e.getValue(), recursive);
+                }
+                return result;
+            }
+        };
+    }
+    private UnitResponse getClusterState(final DummyBackend.Cluster cluster, final int recursive) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() { return null; }
+            @Override
+            public UnitMetrics getMetrics() { return null; }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() {
+                Map<String, SubUnitList> result = new LinkedHashMap<>();
+                SubUnitListImpl subUnits = new SubUnitListImpl();
+                result.put("node", subUnits);
+                for (Map.Entry<String, DummyBackend.Node> e : cluster.nodes.entrySet()) {
+                    subUnits.addUnit(e.getValue(), recursive);
+                }
+                return result;
+            }
+        };
+    }
+    private UnitResponse getNodeState(final DummyBackend.Node node) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() {
+                return new UnitAttributes() {
+                    @Override
+                    public Map<String, String> getAttributeValues() {
+                        Map<String, String> attrs = new LinkedHashMap<>();
+                        attrs.put("group", node.group);
+                        return attrs;
+                    }
+                };
+            }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() {
+                return new CurrentUnitState() {
+                    @Override
+                    public Map<String, UnitState> getStatePerType() {
+                        Map<String, UnitState> m = new LinkedHashMap<>();
+                        m.put("current", new UnitState() {
+                            @Override
+                            public String getId() { return node.state; }
+                            @Override
+                            public String getReason() { return node.reason; }
+                        });
+                        return m;
+                    }
+                };
+            }
+            @Override
+            public UnitMetrics getMetrics() {
+                return new UnitMetrics() {
+                    @Override
+                    public Map<String, Number> getMetricMap() {
+                        Map<String, Number> m = new LinkedHashMap<>();
+                        m.put("doc-count", node.docCount);
+                        return m;
+                    }
+                };
+            }
+        };
+
+    }
+
+    @Override
+    public UnitResponse getState(UnitStateRequest request) throws StateRestApiException {
+        checkForInducedException();
+        String[] path = request.getUnitPath();
+        if (path.length == 0) {
+            return getClusterList(request.getRecursiveLevels());
+        }
+        final DummyBackend.Cluster c = backend.getClusters().get(path[0]);
+        if (c == null) throw new MissingUnitException(path, 0);
+        if (path.length == 1) {
+            return getClusterState(c, request.getRecursiveLevels());
+        }
+        final DummyBackend.Node n = c.nodes.get(path[1]);
+        if (n == null) throw new MissingUnitException(path, 1);
+        if (path.length == 2) {
+            return getNodeState(n);
+        }
+        throw new MissingUnitException(path, 3);
+    }
+
+    @Override
+    public SetResponse setUnitState(SetUnitStateRequest request) throws StateRestApiException {
+        checkForInducedException();
+        String[] path = request.getUnitPath();
+        if (path.length != 2) {
+            throw new OperationNotSupportedForUnitException(
+                    path, "You can only set states on nodes");
+        }
+        DummyBackend.Node n = null;
+        DummyBackend.Cluster c = backend.getClusters().get(path[0]);
+        if (c != null) {
+            n = c.nodes.get(path[1]);
+        }
+        if (n == null) throw new MissingUnitException(path, 2);
+        Map<String, UnitState> newState = request.getNewState();
+        if (newState.size() != 1 || !newState.containsKey("current")) {
+            throw new InvalidContentException("Only state of type 'current' is allowed to be set.");
+        }
+        n.state = newState.get("current").getId();
+        n.reason = newState.get("current").getReason();
+        return new SetResponse("DummyStateAPI", true);
+    }
+
+    private void checkForInducedException() throws StateRestApiException {
+        if (induceException == null) return;
+        Exception e = induceException;
+        induceException = null;
+        if (e instanceof RuntimeException) {
+            throw (RuntimeException) e;
+        }
+        throw (StateRestApiException) e;
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
new file mode 100644
index 00000000000..a76c86fa4a5
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
@@ -0,0 +1,470 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncUtils;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.JsonHttpResult;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.*;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.server.RestApiHandler;
+import com.yahoo.vespa.clustercontroller.utils.test.TestTransport;
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class StateRestAPITest extends TestCase {
+
+    private static void populateDummyBackend(DummyBackend backend) {
+        backend.addCluster(new DummyBackend.Cluster("foo")
+                .addNode(new DummyBackend.Node("1")
+                        .setState("initializing")
+                        .setDocCount(5)
+                )
+                .addNode(new DummyBackend.Node("3")
+                        .setDocCount(8)
+                )
+        ).addCluster(new DummyBackend.Cluster("bar")
+                .addNode(new DummyBackend.Node("2")
+                        .setState("down")
+                )
+        );
+    }
+
+    private DummyStateApi stateApi;
+    private TestTransport testTransport;
+
+    private void setupDummyStateApi() {
+        DummyBackend backend = new DummyBackend();
+        stateApi = new DummyStateApi(backend);
+        populateDummyBackend(backend);
+        testTransport = new TestTransport();
+        RestApiHandler handler = new RestApiHandler(stateApi);
+        handler.setDefaultPathPrefix("/cluster/v2");
+        testTransport.addServer(handler, "host", 80, "/cluster/v2");
+    }
+
+    public void tearDown() {
+        if (testTransport != null) {
+            testTransport.close();
+            testTransport = null;
+        }
+        stateApi = null;
+    }
+
+    private HttpResult execute(HttpRequest request) {
+        request.setHost("host").setPort(80);
+        AsyncOperation<HttpResult> op = testTransport.getClient().execute(request);
+        AsyncUtils.waitFor(op);
+        if (!op.isSuccess()) { // Don't call getCause() unless it fails
+            assertTrue(op.getCause().toString(), op.isSuccess());
+        }
+        assertTrue(op.getResult() != null);
+        return op.getResult();
+    }
+    private JSONObject executeOkJsonRequest(HttpRequest request) {
+        HttpResult result = execute(request);
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        return (JSONObject) result.getContent();
+    }
+
+    public void testTopLevelList() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"cluster\": {\n"
+                        + "  \"foo\": {\"link\": \"\\/cluster\\/v2\\/foo\"},\n"
+                        + "  \"bar\": {\"link\": \"\\/cluster\\/v2\\/bar\"}\n"
+                        + "}}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testClusterState() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"node\": {\n"
+                        + "  \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n"
+                        + "  \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n"
+                        + "}}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testNodeState() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\n"
+                        + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                        + "  \"state\": {\"current\": {\n"
+                        + "    \"state\": \"up\",\n"
+                        + "    \"reason\": \"\"\n"
+                        + "  }},\n"
+                        + "  \"metrics\": {\"doc-count\": 8}\n"
+                        + "}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testRecursiveMode() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "true"));
+            String expected =
+                    "{\"cluster\": {\n" +
+                    "  \"foo\": {\"node\": {\n" +
+                    "    \"1\": {\n" +
+                    "      \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "      \"state\": {\"current\": {\n" +
+                    "        \"state\": \"initializing\",\n" +
+                    "        \"reason\": \"\"\n" +
+                    "      }},\n" +
+                    "      \"metrics\": {\"doc-count\": 5}\n" +
+                    "    },\n" +
+                    "    \"3\": {\n" +
+                    "      \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "      \"state\": {\"current\": {\n" +
+                    "        \"state\": \"up\",\n" +
+                    "        \"reason\": \"\"\n" +
+                    "      }},\n" +
+                    "      \"metrics\": {\"doc-count\": 8}\n" +
+                    "    }\n" +
+                    "  }},\n" +
+                    "  \"bar\": {\"node\": {\"2\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"down\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 0}\n" +
+                    "  }}}\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "1"));
+            String expected =
+                    "{\"cluster\": {\n" +
+                    "  \"foo\": {\"node\": {\n" +
+                    "    \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n" +
+                    "    \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n" +
+                    "  }},\n" +
+                    "  \"bar\": {\"node\": {\"2\": {\"link\": \"\\/cluster\\/v2\\/bar\\/2\"}}}\n" +
+                    "}}";
+            // Verify that the actual link does not contain backslash. It's just an artifact of
+            // jettison json output.
+            assertEquals("/cluster/v2/foo/1",
+                         json.getJSONObject("cluster").getJSONObject("foo").getJSONObject("node")
+                                    .getJSONObject("1").getString("link"));
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2/foo").addUrlOption("recursive", "1"));
+            String expected =
+                    "{\"node\": {\n" +
+                    "  \"1\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"initializing\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 5}\n" +
+                    "  },\n" +
+                    "  \"3\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"up\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 8}\n" +
+                    "  }\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2/foo").addUrlOption("recursive", "false"));
+            String expected =
+                    "{\"node\": {\n" +
+                    "  \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n" +
+                    "  \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+    }
+
+    public void testSetNodeState() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject()
+                            .put("state", "retired")
+                            .put("reason", "No reason")));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(new HttpRequest().setPath("/cluster/v2/foo/3"));
+            String expected = "{\n"
+                    + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                    + "  \"state\": {\"current\": {\n"
+                    + "    \"state\": \"retired\",\n"
+                    + "    \"reason\": \"No reason\"\n"
+                    + "  }},\n"
+                    + "  \"metrics\": {\"doc-count\": 8}\n"
+                    + "}";
+            assertEquals(json.toString(2), expected, json.toString(2));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject()));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(new HttpRequest().setPath("/cluster/v2/foo/3"));
+            String expected = "{\n"
+                    + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                    + "  \"state\": {\"current\": {\n"
+                    + "    \"state\": \"up\",\n"
+                    + "    \"reason\": \"\"\n"
+                    + "  }},\n"
+                    + "  \"metrics\": {\"doc-count\": 8}\n"
+                    + "}";
+            assertEquals(json.toString(2), expected, json.toString(2));
+        }
+        {
+            JSONObject json = new JSONObject()
+                    .put("state", new JSONObject()
+                            .put("current", new JSONObject()
+                                    .put("state", "retired")
+                                    .put("reason", "No reason")))
+                    .put("condition", "FORCE");
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            StringBuilder print = new StringBuilder();
+            result.printContent(print);
+            assertEquals(print.toString(),
+                    "JSON: {\n" +
+                            "  \"wasModified\": true,\n" +
+                            "  \"reason\": \"DummyStateAPI\"\n" +
+                            "}");
+        }
+    }
+
+    public void testMissingUnits() throws Exception {
+        setupDummyStateApi();
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/unknown"));
+            assertEquals(result.toString(true), 404, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "No such resource 'unknown'.", result.getHttpReturnCodeDescription());
+            String expected = "{\"message\":\"No such resource 'unknown'.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/1234"));
+            assertEquals(result.toString(true), 404, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "No such resource 'foo/1234'.", result.getHttpReturnCodeDescription());
+            String expected = "{\"message\":\"No such resource 'foo\\/1234'.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testUnknownMaster() throws Exception {
+        setupDummyStateApi();
+        stateApi.induceException(new UnknownMasterException());
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 503, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "Service Unavailable", result.getHttpReturnCodeDescription());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"message\":\"No known master cluster controller currently exists.\"}";
+        assertEquals(expected, result.getContent().toString());
+        assertTrue(result.getHeader("Location") == null);
+    }
+
+    public void testOtherMaster() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new OtherMasterException("example.com", 80));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption(" %=?&", "&?%=").addUrlOption("foo", "bar"));
+            assertEquals(result.toString(true), 307, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Temporary Redirect", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "http://example.com:80/cluster/v2?%20%25%3D%3F%26=%26%3F%25%3D&foo=bar", result.getHeader("Location"));
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Cluster controller not master. Use master at example.com:80.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            stateApi.induceException(new OtherMasterException("example.com", 80));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo"));
+            assertEquals(result.toString(true), 307, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Temporary Redirect", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "http://example.com:80/cluster/v2/foo", result.getHeader("Location"));
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Cluster controller not master. Use master at example.com:80.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testRuntimeException() throws Exception {
+        setupDummyStateApi();
+        stateApi.induceException(new RuntimeException("Moahaha"));
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "Failed to process request", result.getHttpReturnCodeDescription());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"message\":\"java.lang.RuntimeException: Moahaha\"}";
+        assertEquals(expected, result.getContent().toString());
+    }
+
+    public void testClientFailures() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new InvalidContentException("Foo bar"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Foo bar\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            stateApi.induceException(new InvalidOptionValueException("foo", "bar", "Foo can not be bar"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'foo' have invalid value 'bar'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'foo' have invalid value 'bar': Foo can not be bar\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            String path[] = new String[1];
+            path[0] = "foo";
+            stateApi.induceException(new OperationNotSupportedForUnitException(path, "Foo"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 405, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Operation not supported for resource", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"[foo]: Foo\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInternalFailure() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new InternalFailure("Foo"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Failed to process request", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Internal failure. Should not happen: Foo\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInvalidRecursiveValues() throws Exception {
+        setupDummyStateApi();
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "-5"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'recursive' have invalid value '-5'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'recursive' have invalid value '-5': Recursive option must be true, false, 0 or a positive integer\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "foo"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'recursive' have invalid value 'foo'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'recursive' have invalid value 'foo': Recursive option must be true, false, 0 or a positive integer\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInvalidJsonInSetStateRequest() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = new JSONObject();
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("Set state requests must contain a state object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", 5);
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state is not a json object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", 5));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current is not a json object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject().put("state", 5)));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current->state is not a string"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject().put("state", "down").put("reason", 5)));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current->reason is not a string"));
+        }
+        {
+            JSONObject json = new JSONObject()
+                    .put("state", new JSONObject()
+                            .put("current", new JSONObject()
+                                    .put("state", "retired")
+                                    .put("reason", "No reason")))
+                    .put("condition", "Non existing condition");
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            StringBuilder print = new StringBuilder();
+            result.printContent(print);
+            assertEquals(print.toString(),
+                    "JSON: {\"message\": \"java.lang.IllegalArgumentException: No enum constant " +
+                            "com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest." +
+                            "Condition.Non existing condition\"}");
+        }
+    }
+
+    public void testInvalidPathPrefix() throws Exception {
+        DummyBackend backend = new DummyBackend();
+        stateApi = new DummyStateApi(backend);
+        populateDummyBackend(backend);
+        testTransport = new TestTransport();
+        RestApiHandler handler = new RestApiHandler(stateApi);
+        try{
+            handler.setDefaultPathPrefix("cluster/v2");
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java
new file mode 100644
index 00000000000..173f6c8704a
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import junit.framework.TestCase;
+
+public class FakeClockTest extends TestCase {
+
+    public void testSimple() {
+        FakeClock clock = new FakeClock();
+            // Should not start at 0, as that is common not initialized yet value
+        assertTrue(clock.getTimeInMillis() > 0);
+        long start = clock.getTimeInMillis();
+
+        clock.adjust(5);
+        assertEquals(start + 5, clock.getTimeInMillis());
+
+        clock.set(start + 10);
+        assertEquals(start + 10, clock.getTimeInMillis());
+
+        clock.adjust(5);
+        assertEquals(start + 15, clock.getTimeInMillis());
+    }
+
+    /**
+     * @todo This should probably throw exceptions.. However, that doesn't seem to be current behavior. I suspect some tests misuse the clock to reset things to run another test. Should probably be fixed.
+     */
+    public void testTurnTimeBack() {
+        FakeClock clock = new FakeClock();
+        clock.set(1000);
+
+        clock.set(500);
+        assertEquals(500, clock.getTimeInMillis());
+
+        clock.adjust(-100);
+        assertEquals(400, clock.getTimeInMillis());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java
new file mode 100644
index 00000000000..e2dac056cad
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class CertainlyCloneableTest extends TestCase {
+
+    private class Foo extends CertainlyCloneable<Foo> {
+        protected Foo callParentClone() throws CloneNotSupportedException {
+            throw new CloneNotSupportedException("Foo");
+        }
+    }
+
+    public void testSimple() {
+        try{
+            Foo f = new Foo();
+            f.clone();
+            fail("Control should not get here");
+        } catch (Error e) {
+            assertEquals("Foo", e.getCause().getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java
new file mode 100644
index 00000000000..4282b1f2020
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class ClockTest extends TestCase {
+
+    public void testNothingButGetCoverage() {
+        long s = new Clock().getTimeInSecs();
+        long ms = new Clock().getTimeInMillis();
+        assertTrue(ms >= 1000 * s);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java
new file mode 100644
index 00000000000..160ef498023
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class JSONObjectWrapperTest extends TestCase {
+
+    public void testExceptionWrapping() {
+        JSONObjectWrapper wrapper = new JSONObjectWrapper();
+        try{
+            wrapper.put(null, "foo");
+        } catch (NullPointerException e) {
+            assertEquals("Null key.", e.getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java
new file mode 100644
index 00000000000..1b3ffdc78e6
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class MetricReporterTest extends TestCase {
+    static class MetricReporterMock implements MetricReporter {
+        StringBuilder sb = new StringBuilder();
+
+        @Override
+        public void set(String s, Number number, Context context) {
+            sb.append("set(").append(s).append(", ").append(number).append(")\n");
+        }
+
+        @Override
+        public void add(String s, Number number, Context context) {
+            sb.append("add(").append(s).append(", ").append(number).append(")\n");
+        }
+        @Override
+        public Context createContext(Map<String, ?> stringMap) {
+            sb.append("createContext(");
+            for (String s : stringMap.keySet()) {
+                sb.append(" ").append(s).append("=").append(stringMap.get(s));
+            }
+            sb.append(" )\n");
+            return new Context() {};
+        }
+    };
+
+    public void testNoMetricReporter() {
+        NoMetricReporter reporter = new NoMetricReporter();
+        reporter.add("foo", 3, null);
+        reporter.set("foo", 3, null);
+        reporter.createContext(null);
+    }
+
+    public void testPrefix() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        c.add("foo", 2);
+        c.set("bar", 1);
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "add(prefixfoo, 2)\n" +
+                "set(prefixbar, 1)\n", mock.sb.toString());
+
+    }
+
+    public void testWithContext() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        Map<String, Integer> myContext = new TreeMap<>();
+        myContext.put("myvar", 3);
+        c.add("foo", 2, c.createContext(myContext));
+        c.set("bar", 1, c.createContext(myContext));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "createContext( myvar=3 urk=fy )\n" +
+                "add(prefixfoo, 2)\n" +
+                "createContext( myvar=3 urk=fy )\n" +
+                "set(prefixbar, 1)\n", mock.sb.toString());
+    }
+
+    public void testDefaultContext() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        c.add("foo", 2, c.createContext(null));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "add(prefixfoo, 2)\n", mock.sb.toString());
+    }
+
+    public void testContextOverlap() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        Map<String, String> myContext = new TreeMap<>();
+        myContext.put("urk", "yes");
+        c.add("foo", 2, c.createContext(myContext));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "createContext( urk=yes )\n" +
+                "add(prefixfoo, 2)\n", mock.sb.toString());
+    }
+
+}