Publish

62 files changed, 2188 insertions, 0 deletions

diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java
new file mode 100644
index 00000000000..a68b90b6eef
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncCallback.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public interface AsyncCallback<T>   {
+    /** Callback indicating the given operation has completed. */
+    public void done(AsyncOperation<T> op);
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java
new file mode 100644
index 00000000000..3099f57ba09
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperation.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public interface AsyncOperation<T>   {
+    /**
+     * Attempt to cancel the given operation.
+     * @return True if successfully cancelled. False if cancel is not supported, or operation already succeeded.
+     */
+    public boolean cancel();
+
+    /**
+     * Register a callback to be called when this operation completes. If operation is already completed, this callback
+     * will be called immediately upon registering. The same callback should not be registered multiple times. It is
+     * suggested to throw an exception if that should happen. Otherwise you may get one or more calls to that component.
+     */
+    void register(AsyncCallback<T> callback);
+
+    /**
+     * Remove a callback from the list to be called when operation is completed. If callback has not already been called
+     * at the time this function returns, it should never be called by this operation, unless re-registered.
+     */
+    void unregister(AsyncCallback<T> callback);
+
+    /**
+     * Get the name of the operation. Useful to identify what operation this is.
+     */
+    public String getName();
+
+    /**
+     * Get a description of the operation. May be empty. If operation is complex one might want to use a short name for
+     * simplicity, but have the whole request available if needed. In the HTTP case an application may for instance include
+     * the URL in the name, and add the request headers to the description.
+     */
+    public String getDescription();
+
+    /**
+     * Get the progress as a number between 0 and 1 where 0 means not started and 1 means operation is complete.
+     * A return value of null indicates that the operation is unable to track progress.
+     */
+    public Double getProgress();
+
+    /**
+     * Get the result of the operation.
+     * Note that some operations may not have a result if the operation failed.
+     */
+    public T getResult();
+
+    /** Get the cause of an operation failing. Returns null on successful operations. */
+    public Exception getCause();
+
+    /** Returns true if operation has been successfully cancelled. */
+    public boolean isCanceled();
+    /** Returns true if operation has completed. Regardless of whether it was a success or a failure. */
+    public boolean isDone();
+    /** Returns true if the operation was a success. */
+    public boolean isSuccess();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java
new file mode 100644
index 00000000000..79da91d40b2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationImpl.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import java.util.logging.Logger;
+
+public class AsyncOperationImpl<T> implements AsyncOperation<T> {
+    private static final Logger log = Logger.getLogger(AsyncOperationImpl.class.getName());
+    private final String name;
+    private final String description;
+    private boolean resultGotten = false; // Ensures result is set only once.
+    private boolean completed = false; // Indicates that operation is complete.
+    private boolean failed = false;
+    private T result;
+    private Exception failure;
+    private AsyncOperationListenImpl<T> listenImpl;
+
+    public AsyncOperationImpl(String name) {
+        this(name, null);
+    }
+    public AsyncOperationImpl(String name, String description) {
+        this.name = name;
+        this.description = description;
+        listenImpl = new AsyncOperationListenImpl<T>(this);
+    }
+
+    private boolean tagResultHandled() {
+        synchronized (listenImpl) {
+            if (resultGotten) {
+                log.fine("Operation " + this + " got result attempted set twice. This may occasionally happen if multiple "
+                       + "sources are set to possibly terminate operations, such as for example if there is a separate cancel or timeout "
+                       + "handler.");
+                return false;
+            }
+            resultGotten = true;
+            return true;
+        }
+    }
+
+    public void setFailure(Exception e) { setFailure(e, null); }
+    public void setFailure(Exception e, T partialResult) {
+        if (!tagResultHandled()) { return; }
+        failed = true;
+        failure = e;
+        this.result = partialResult;
+        completed = true;
+        listenImpl.notifyListeners();
+    }
+    public void setResult(T result) {
+        if (!tagResultHandled()) return;
+        this.result = result;
+        completed = true;
+        listenImpl.notifyListeners();
+    }
+
+    @Override
+    public String getName() { return name; }
+    @Override
+    public String getDescription() { return description; }
+    @Override
+    public String toString() { return "AsyncOperationImpl(" + name + ")"; }
+    @Override
+    public T getResult() { return result; }
+    @Override
+    public boolean cancel() { return false; }
+    @Override
+    public boolean isCanceled() { return false; }
+    @Override
+    public boolean isDone() { return completed; }
+    @Override
+    public boolean isSuccess() { return (completed && !failed); }
+    @Override
+    public Double getProgress() { return (completed ? 1.0 : null); }
+    @Override
+    public Exception getCause() { return failure; }
+    @Override
+    public void register(AsyncCallback<T> callback) {
+        listenImpl.register(callback);
+    }
+    @Override
+    public void unregister(AsyncCallback<T> callback) {
+        listenImpl.unregister(callback);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java
new file mode 100644
index 00000000000..e9007f0c16e
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncOperationListenImpl.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.logging.Logger;
+
+public class AsyncOperationListenImpl<T> {
+    private static final Logger log = Logger.getLogger(AsyncOperationListenImpl.class.getName());
+    private final Collection<AsyncCallback<T>> listeners = new HashSet<>();
+    private boolean listenersNotified = false;
+    private AsyncOperation<T> op;
+
+    protected AsyncOperationListenImpl(AsyncOperation<T> op) {
+        this.op = op;
+    }
+
+    public void register(AsyncCallback<T> callback) {
+        synchronized (listeners) {
+            listeners.add(callback);
+            if (listenersNotified) callback.done(op);
+        }
+    }
+    public void unregister(AsyncCallback<T> callback) {
+        synchronized (listeners) {
+            listeners.remove(callback);
+        }
+    }
+
+    public void notifyListeners() {
+        synchronized (listeners) {
+            if (listenersNotified) return;
+            for(AsyncCallback<T> callback : listeners) {
+                try{
+                    callback.done(op);
+                } catch (RuntimeException e) {
+                    StringWriter sw = new StringWriter();
+                    e.printStackTrace(new PrintWriter(sw));
+                    log.warning("Callback '" + callback + "' threw exception on notify. Should not happen:\n" + sw);
+                }
+            }
+            listenersNotified = true;
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java
new file mode 100644
index 00000000000..71d6f534a9a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncUtils.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public class AsyncUtils {
+
+    public static void waitFor(AsyncOperation op) {
+        while (!op.isDone()) {
+            try{ Thread.sleep(1); } catch (InterruptedException e) {}
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java
new file mode 100644
index 00000000000..a7ce5bb61a2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/PipedAsyncOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public abstract class PipedAsyncOperation<S, T> extends RedirectedAsyncOperation<S, T> {
+    private T result;
+
+    public PipedAsyncOperation(AsyncOperation<S> source) {
+        super(source);
+        setOnCompleteTask(new AsyncCallback<S>() {
+            @Override
+            public void done(AsyncOperation<S> op) {
+                result = convertResult(op.getResult());
+            }
+        });
+    }
+
+    public abstract T convertResult(S result);
+
+    @Override
+    public T getResult() {
+        return result;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java
new file mode 100644
index 00000000000..fdaf0dcbf2b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/RedirectedAsyncOperation.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+/**
+ * Utility class in order to wrap com.yahoo.vespa.clustercontroller.utils.communication.async operation callbacks. Useful when translating com.yahoo.vespa.clustercontroller.utils.communication.async operations returning JSON to com.yahoo.vespa.clustercontroller.utils.communication.async operations returning specific values.
+ */
+public abstract class RedirectedAsyncOperation<S, T> implements AsyncOperation<T> {
+    protected final AsyncOperation<S> source;
+    private final AsyncOperationListenImpl<T> listenImpl;
+    private AsyncCallback<S> beforeCallbackTask;
+
+    public RedirectedAsyncOperation(AsyncOperation<S> source) {
+        this.source = source;
+        this.listenImpl = new AsyncOperationListenImpl<>(this);
+        source.register(new AsyncCallback<S>() {
+            @Override
+            public void done(AsyncOperation<S> op) { notifyDone(); }
+        });
+    }
+
+    public RedirectedAsyncOperation<S, T> setOnCompleteTask(AsyncCallback<S> beforeTask) {
+        beforeCallbackTask = beforeTask;
+        return this;
+    }
+    private void notifyDone() {
+        if (beforeCallbackTask != null) beforeCallbackTask.done(source);
+        listenImpl.notifyListeners();
+    }
+
+    @Override
+    public String getName() { return source.getName(); }
+
+    @Override
+    public String getDescription() { return source.getDescription(); }
+
+    @Override
+    public boolean cancel() { return source.cancel(); }
+
+    @Override
+    public boolean isCanceled() { return source.isCanceled(); }
+
+    @Override
+    public boolean isDone() { return source.isDone(); }
+
+    @Override
+    public boolean isSuccess() { return source.isSuccess(); }
+
+    @Override
+    public Double getProgress() { return source.getProgress(); }
+
+    @Override
+    public Exception getCause() { return source.getCause(); }
+
+    @Override
+    public void register(AsyncCallback<T> callback) {
+        listenImpl.register(callback);
+    }
+
+    @Override
+    public void unregister(AsyncCallback<T> callback) {
+        listenImpl.unregister(callback);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java
new file mode 100644
index 00000000000..0572465aac5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/SuccessfulAsyncCallback.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+public abstract class SuccessfulAsyncCallback<Source, Target> implements AsyncCallback<Source> {
+    private final AsyncOperationImpl<Target> target;
+
+    public SuccessfulAsyncCallback(final AsyncOperationImpl<Target> target) {
+        this.target = target;
+    }
+
+    public void done(AsyncOperation<Source> sourceOp) {
+        if (sourceOp.isSuccess()) {
+            successfullyDone(sourceOp);
+        } else {
+            target.setFailure(sourceOp.getCause());
+        }
+    }
+
+    public abstract void successfullyDone(AsyncOperation<Source> op);
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java
new file mode 100644
index 00000000000..205da8cf98f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/async/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java
new file mode 100644
index 00000000000..5a5dd21c8f5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClient.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+/**
+ * Abstraction of an asynchronious HTTP client, such that applications don't need to depend directly on an HTTP client.
+ */
+public interface AsyncHttpClient<V extends HttpResult> {
+
+    public AsyncOperation<V> execute(HttpRequest r);
+
+    /** Attempt to cancel all pending operations and shut down the client. */
+    public void close();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java
new file mode 100644
index 00000000000..419c19f529b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBase.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+public class AsyncHttpClientWithBase<V extends HttpResult> implements AsyncHttpClient<V> {
+    protected final AsyncHttpClient<V> client;
+    private HttpRequest baseRequest = new HttpRequest();
+
+    public AsyncHttpClientWithBase(AsyncHttpClient<V> client) {
+        if (client == null) throw new IllegalArgumentException("HTTP client must be set.");
+        this.client = client;
+    }
+
+    /**
+     * If all your http requests have common features you want to set once, you can provide those values in a base
+     * request. For instance, if you specify a host and a port using this function, all your requests will use that
+     * host and port unless specified in the request you execute.
+     */
+    public void setHttpRequestBase(HttpRequest r) {
+        this.baseRequest = (r == null ? new HttpRequest() : r.clone());
+    }
+
+    public HttpRequest getHttpRequestBase() {
+        return baseRequest;
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        return client.execute(baseRequest.merge(r));
+    }
+
+    @Override
+    public void close() {
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java
new file mode 100644
index 00000000000..c64da81cac1
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequest.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.util.CertainlyCloneable;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class HttpRequest extends CertainlyCloneable<HttpRequest> {
+    public static class KeyValuePair {
+        public String key;
+        public String value;
+
+        public KeyValuePair(String k, String v) { this.key = k; this.value = v; }
+
+        public String getValue() {
+            return value;
+        }
+
+        public String getKey() {
+            return key;
+        }
+    }
+    public enum HttpOp { GET, POST, PUT, DELETE }
+
+    private String host;
+    private int port;
+    private String path;
+    private List<KeyValuePair> urlOptions = new LinkedList<>();
+    private List<KeyValuePair> headers = new LinkedList<>();
+    private long timeoutMillis;
+    private Object postContent;
+    private HttpOp httpOperation;
+
+    public HttpRequest() {}
+
+    public String getHost() { return host; }
+    public int getPort() { return port; }
+    public String getPath() { return path; }
+    public List<KeyValuePair> getUrlOptions() { return urlOptions; }
+    public String getOption(String key, String defaultValue) {
+        for (KeyValuePair value : urlOptions) {
+            if (value.key.equals(key)) return value.value;
+        }
+        return defaultValue;
+
+    }
+    public List<KeyValuePair> getHeaders() { return headers; }
+    public String getHeader(String key, String defaultValue) {
+        for (KeyValuePair header : headers) {
+            if (header.key.equals(key)) return header.value;
+        }
+        return defaultValue;
+    }
+    public long getTimeoutMillis() { return timeoutMillis; }
+    public Object getPostContent() { return postContent; }
+    public HttpOp getHttpOperation() { return httpOperation; }
+
+    public HttpRequest setHost(String hostname) { this.host = hostname; return this; }
+    public HttpRequest setPort(int port) { this.port = port; return this; }
+    public HttpRequest setPath(String path) {
+        this.path = path; return this;
+    }
+    public HttpRequest addUrlOption(String key, String value) { this.urlOptions.add(new KeyValuePair(key, value)); return this; }
+    public HttpRequest setUrlOptions(List<KeyValuePair> options) { this.urlOptions.clear(); this.urlOptions.addAll(options); return this; }
+    public HttpRequest addHttpHeader(String key, String value) { this.headers.add(new KeyValuePair(key, value)); return this; }
+    public HttpRequest setTimeout(long timeoutMillis) { this.timeoutMillis = timeoutMillis; return this; }
+    public HttpRequest setPostContent(Object content) { this.postContent = content; return this; }
+    public HttpRequest setHttpOperation(HttpOp op) { this.httpOperation = op; return this; }
+
+    /** Create a copy of this request, and override what is specified in the input in the new request. */
+    public HttpRequest merge(HttpRequest r) {
+        HttpRequest copy = clone();
+        if (r.host != null) copy.host = r.host;
+        if (r.port != 0) copy.port = r.port;
+        if (r.path != null) copy.path = r.path;
+        for (KeyValuePair h : r.headers) {
+            boolean containsElement = false;
+            for (KeyValuePair h2 : copy.headers) { containsElement |= (h.key.equals(h2.key)); }
+            if (!containsElement) copy.headers.add(h);
+        }
+        for (KeyValuePair h : r.urlOptions) {
+            boolean containsElement = false;
+            for (KeyValuePair h2 : copy.urlOptions) { containsElement |= (h.key.equals(h2.key)); }
+            if (!containsElement) copy.urlOptions.add(h);
+        }
+        if (r.timeoutMillis != 0) copy.timeoutMillis = r.timeoutMillis;
+        if (r.postContent != null) copy.postContent = r.postContent;
+        if (r.httpOperation != null) copy.httpOperation = r.httpOperation;
+        return copy;
+    }
+
+    @Override
+    public HttpRequest clone() {
+        HttpRequest r = (HttpRequest) super.clone();
+        r.headers = new LinkedList<>(r.headers);
+        r.urlOptions = new LinkedList<>(r.urlOptions);
+        return r;
+    }
+
+    @Override
+    public String toString() { return toString(false); }
+    public String toString(boolean verbose) {
+        String httpOp = (httpOperation != null ? httpOperation.toString()
+                                               : (postContent == null ? "GET?" : "POST?"));
+        StringBuilder sb = new StringBuilder().append(httpOp).append(" http:");
+        if (host != null) {
+            sb.append("//").append(host);
+            if (port != 0) sb.append(':').append(port);
+        }
+        if (path == null || path.isEmpty()) {
+            sb.append('/');
+        } else {
+            if (path.charAt(0) != '/') sb.append('/');
+            sb.append(path);
+        }
+        if (urlOptions != null && urlOptions.size() > 0) {
+            boolean first = (path == null || path.indexOf('?') < 0);
+            for (KeyValuePair e : urlOptions) {
+                sb.append(first ? '?' : '&');
+                first = false;
+                sb.append(e.key).append('=').append(e.value);
+            }
+        }
+        if (verbose) {
+            for (KeyValuePair p : headers) {
+                sb.append('\n').append(p.key).append(": ").append(p.value);
+            }
+            if (postContent != null && !postContent.toString().isEmpty()) {
+                sb.append("\n\n").append(postContent.toString());
+            }
+        }
+        return sb.toString();
+    }
+
+    public void verifyComplete() {
+        if (path == null) throw new IllegalStateException("HTTP requests must have a path set. Use '/' for top level");
+        if (httpOperation == null) throw new IllegalStateException("HTTP requests must have an HTTP method defined");
+    }
+
+    public boolean equals(Object o) {
+        // Equals is only used for debugging as far as we know. Refer to verbose toString to simplify
+        if (o instanceof HttpRequest) {
+            return toString(true).equals(((HttpRequest) o).toString(true));
+        }
+        return false;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java
new file mode 100644
index 00000000000..b8acfa33e71
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestHandler.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+public interface HttpRequestHandler {
+
+    public HttpResult handleRequest(HttpRequest request) throws Exception;
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java
new file mode 100644
index 00000000000..7e2bf90b92c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResult.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+public class HttpResult {
+    private static class HttpReturnCode {
+        private final int code;
+        private final String message;
+
+        public HttpReturnCode(int code, String message) {
+            this.code = code;
+            this.message = message;
+        }
+        public boolean isSuccess() { return (code >= 200 && code < 300); }
+        public int getCode() { return code; }
+        public String getMessage() { return message; }
+    }
+    private HttpReturnCode httpReturnCode;
+    private final List<HttpRequest.KeyValuePair> headers;
+    private Object content;
+
+    public HttpResult() {
+        httpReturnCode = new HttpReturnCode(200, "OK");
+        headers = new LinkedList<>();
+    }
+
+    public HttpResult(HttpResult other) {
+        httpReturnCode = other.httpReturnCode;
+        headers = other.headers;
+        content = other.content;
+    }
+
+    public HttpResult setHttpCode(int code, String description) {
+        this.httpReturnCode = new HttpReturnCode(code, description);
+        return this;
+    }
+
+    public HttpResult setContent(Object content) {
+        this.content = content;
+        return this;
+    }
+
+    public HttpResult addHeader(String key, String value) {
+        headers.add(new HttpRequest.KeyValuePair(key, value));
+        return this;
+    }
+
+    public boolean isSuccess() { return httpReturnCode.isSuccess(); }
+    public int getHttpReturnCode() { return httpReturnCode.getCode(); }
+    public String getHttpReturnCodeDescription() { return httpReturnCode.getMessage(); }
+    public Collection<HttpRequest.KeyValuePair> getHeaders() { return headers; }
+    public String getHeader(String key) {
+        for (HttpRequest.KeyValuePair p : headers) {
+            if (p.getKey().equals(key)) return p.getValue();
+        }
+        return null;
+    }
+
+    public Object getContent() { return content; }
+
+    @Override
+    public String toString() { return toString(false); }
+    public String toString(boolean verbose) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("HTTP ").append(httpReturnCode.getCode()).append('/').append(httpReturnCode.getMessage());
+        if (verbose) {
+            for(HttpRequest.KeyValuePair header : headers) {
+                sb.append('\n').append(header.getKey()).append(": ").append(header.getValue());
+            }
+            if (content != null) {
+                StringBuilder contentBuilder = new StringBuilder();
+                printContent(contentBuilder);
+                String s = contentBuilder.toString();
+                if (!s.isEmpty()) {
+                    sb.append("\n\n").append(s);
+                }
+            }
+        }
+        return sb.toString();
+    }
+    public void printContent(StringBuilder sb) {
+        sb.append(content.toString());
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java
new file mode 100644
index 00000000000..bffac7f68e3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClient.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.RedirectedAsyncOperation;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * Wrapped for the HTTP client, converting requests to/from JSON.
+ */
+public class JsonAsyncHttpClient implements AsyncHttpClient<JsonHttpResult> {
+    private AsyncHttpClient<HttpResult> client;
+    private boolean verifyRequestContentAsJson = true;
+    private boolean addJsonContentType = true;
+
+    public JsonAsyncHttpClient(AsyncHttpClient<HttpResult> client) {
+        this.client = client;
+    }
+
+    public JsonAsyncHttpClient verifyRequestContentAsJson(boolean doIt) {
+        verifyRequestContentAsJson = doIt;
+        return this;
+    }
+
+    public JsonAsyncHttpClient addJsonContentType(boolean doIt) {
+        addJsonContentType = doIt;
+        return this;
+    }
+
+    public AsyncOperation<JsonHttpResult> execute(HttpRequest r) {
+        if (verifyRequestContentAsJson) {
+            if (r.getPostContent() != null && !(r.getPostContent() instanceof JSONObject)) {
+                try{
+                    r = r.clone().setPostContent(new JSONObject(r.getPostContent().toString()));
+                } catch (JSONException e) {
+                    throw new IllegalArgumentException(e);
+                }
+            }
+        }
+        if (addJsonContentType && r.getPostContent() != null) {
+            r = r.clone().addHttpHeader("Content-Type", "application/json");
+        }
+        final AsyncOperation<HttpResult> op = client.execute(r);
+        return new RedirectedAsyncOperation<HttpResult, JsonHttpResult>(op) {
+            @Override
+            public JsonHttpResult getResult() {
+                return (op.getResult() == null ? null : new JsonHttpResult(op.getResult()));
+            }
+        };
+    }
+
+    @Override
+    public void close() {
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java
new file mode 100644
index 00000000000..e26ea4fc3a2
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResult.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.util.JSONObjectWrapper;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonHttpResult extends HttpResult {
+    private JSONObject json;
+    private boolean failedParsing = false;
+
+
+    public JsonHttpResult() {
+        addHeader("Content-Type", "application/json");
+    }
+
+    public JsonHttpResult(HttpResult other) {
+        super(other);
+
+        if (other.getContent() == null) {
+            setParsedJson(new JSONObject());
+            return;
+        }
+        try{
+            if (other.getContent() instanceof JSONObject) {
+                setParsedJson((JSONObject) other.getContent());
+            } else {
+                setParsedJson(new JSONObject(other.getContent().toString()));
+            }
+        } catch (JSONException e) {
+            failedParsing = true;
+            setParsedJson(createErrorJson(e.getMessage(), other));
+        }
+    }
+
+    private JSONObject createErrorJson(String error, HttpResult other) {
+        return new JSONObjectWrapper()
+                .put("error", "Invalid JSON in output: " + error)
+                .put("output", other.getContent().toString());
+    }
+
+    public JsonHttpResult setJson(JSONObject o) {
+        setContent(o);
+        json = o;
+        return this;
+    }
+
+    private void setParsedJson(JSONObject o) {
+        json = o;
+    }
+
+    public JSONObject getJson() {
+        return json;
+    }
+
+    @Override
+    public void printContent(StringBuilder sb) {
+        if (failedParsing) {
+            super.printContent(sb);
+            return;
+        }
+        if (json != null) {
+            sb.append("JSON: ");
+            try{
+                sb.append(json.toString(2));
+            } catch (JSONException e) {
+                sb.append(json.toString());
+            }
+        } else {
+            super.printContent(sb);
+        }
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java
new file mode 100644
index 00000000000..d4aff12146d
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClient.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.PipedAsyncOperation;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class LoggingAsyncHttpClient<T extends HttpResult> extends AsyncHttpClientWithBase<T> {
+    private static final Logger log = Logger.getLogger(LoggingAsyncHttpClient.class.getName());
+    private int requestCounter = 0;
+
+    public LoggingAsyncHttpClient(AsyncHttpClient<T> client) {
+        super(client);
+        log.info("Logging HTTP requests if fine logging level is added");
+    }
+
+    public AsyncOperation<T> execute(HttpRequest r) {
+        final int requestCount = ++requestCounter;
+        log.fine("Issuing HTTP request " + requestCount + ": " + r.toString(true));
+        final AsyncOperation<T> op = client.execute(r);
+        return new PipedAsyncOperation<T, T>(op) {
+            @Override
+            public T convertResult(T result) {
+                if (log.isLoggable(Level.FINE)) {
+                    if (op.isSuccess()) {
+                        log.fine("HTTP request " + requestCount + " completed: " + result.toString(true));
+                    } else {
+                        StringWriter sw = new StringWriter();
+                        op.getCause().printStackTrace(new PrintWriter(sw));
+                        log.fine("HTTP request " + requestCount + " failed: " + sw);
+                    }
+                }
+                return result;
+            }
+        };
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java
new file mode 100644
index 00000000000..6b63027d0af
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClient.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+public class ProxyAsyncHttpClient<V extends HttpResult> extends AsyncHttpClientWithBase<V> {
+    private final String proxyHost;
+    private final int proxyPort;
+
+    public ProxyAsyncHttpClient(AsyncHttpClient<V> client, String proxyHost, int proxyPort) {
+        super(client);
+        this.proxyHost = proxyHost;
+        this.proxyPort = proxyPort;
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        r = getHttpRequestBase().merge(r);
+        if (r.getHost() == null || r.getPath() == null) {
+            throw new IllegalStateException("Host and path must be set prior to being able to proxy an HTTP request");
+        }
+        StringBuilder path = new StringBuilder().append(r.getHost());
+        if (r.getPort() != 0) path.append(':').append(r.getPort());
+        if (r.getPath().isEmpty() || r.getPath().charAt(0) != '/') path.append('/');
+        path.append(r.getPath());
+        return client.execute(r.setHost(proxyHost).setPort(proxyPort).setPath(path.toString()));
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java
new file mode 100644
index 00000000000..6388d32f98d
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueue.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+
+import java.util.LinkedList;
+import java.util.logging.Logger;
+
+/**
+ * Utility class to schedule HTTP requests and keeping a maximum amount of them pending at a time.
+ */
+public class RequestQueue<V extends HttpResult> {
+    private static final Logger log = Logger.getLogger(RequestQueue.class.getName());
+    private final AsyncHttpClient<V> httpClient;
+    private final LinkedList<Request<V>> requestQueue = new LinkedList<>();
+    private final int maxPendingRequests;
+    private int pendingRequests = 0;
+
+    public RequestQueue(AsyncHttpClient<V> httpClient, int maxPendingRequests) {
+        this.httpClient = httpClient;
+        this.maxPendingRequests = maxPendingRequests;
+    }
+
+    public boolean empty() {
+        synchronized (requestQueue) {
+            return (requestQueue.isEmpty() && pendingRequests == 0);
+        }
+    }
+
+    public void waitUntilEmpty() throws InterruptedException {
+        synchronized (requestQueue) {
+            while (!empty()) {
+                requestQueue.wait();
+            }
+        }
+    }
+
+    public void schedule(HttpRequest request, AsyncCallback<V> callback) {
+        log.fine("Scheduling " + request + " call");
+        synchronized (requestQueue) {
+            requestQueue.addLast(new Request<>(request, callback));
+            sendMore();
+        }
+    }
+
+    private void sendMore() {
+        while (pendingRequests < maxPendingRequests && !requestQueue.isEmpty()) {
+            Request<V> call = requestQueue.removeFirst();
+            log.fine("Sending " + call.getRequest() + ".");
+            ++pendingRequests;
+            AsyncOperation<V> op = httpClient.execute(call.getRequest());
+            op.register(call);
+        }
+    }
+
+    private class Request<V extends HttpResult> implements AsyncCallback<V> {
+        private final HttpRequest request;
+        private final AsyncCallback<V> callback;
+
+        Request(HttpRequest request, AsyncCallback<V> callback) {
+            this.request = request;
+            this.callback = callback;
+        }
+
+        public HttpRequest getRequest() { return request; }
+
+        @Override
+        public void done(AsyncOperation<V> op) {
+            if (op.isSuccess()) {
+                log.fine("Operation " + op.getName() + " completed successfully");
+            } else {
+                log.fine("Operation " + op.getName() + " failed: " + op.getCause());
+            }
+            synchronized (requestQueue) {
+                --pendingRequests;
+            }
+            callback.done(op);
+            synchronized (requestQueue) {
+                requestQueue.notifyAll();
+                sendMore();
+            }
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java
new file mode 100644
index 00000000000..cdb20be3338
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/SyncHttpClient.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+public interface SyncHttpClient {
+    public HttpResult execute(HttpRequest r);
+
+    /** Attempt to cancel all pending operations and shut down the client. */
+    public void close();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java
new file mode 100644
index 00000000000..c406dafe063
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandler.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.util.Clock;
+
+import java.util.*;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+
+public class TimeoutHandler<V extends HttpResult> extends AsyncHttpClientWithBase<V> {
+    public static class InternalRequest<V extends HttpResult> extends AsyncOperationImpl<V> {
+        final AsyncOperation<V> operation;
+        long startTime;
+        long timeout;
+
+        public InternalRequest(AsyncOperation<V> op, long startTime, long timeout) {
+            super(op.getName(), op.getDescription());
+            this.operation = op;
+            this.startTime = startTime;
+            this.timeout = timeout;
+            op.register(new AsyncCallback<V>() {
+                @Override
+                public void done(AsyncOperation<V> op) {
+                    if (!isDone()) {
+                        if (op.isSuccess()) {
+                            setResult(op.getResult());
+                        } else {
+                            setFailure(op.getCause(), op.getResult());
+                        }
+                    }
+                }
+            });
+        }
+
+        public long getTimeoutTime() { return startTime + timeout; }
+
+        public void handleTimeout(long currentTime) {
+            long timePassed = currentTime - startTime;
+            this.setFailure(new TimeoutException("Operation timeout. " + timePassed + " ms since operation was issued. Timeout was " + timeout + " ms."));
+            operation.cancel();
+        }
+
+        @Override
+        public boolean cancel() { return operation.cancel(); }
+        @Override
+        public boolean isCanceled() { return operation.isCanceled(); }
+        @Override
+        public Double getProgress() { return (isDone() ? Double.valueOf(1.0) : operation.getProgress()); }
+    }
+
+    public static class ChangeLogger {
+        private InternalRequest lastTimeoutLogged = null;
+        private boolean emptyLogged = true;
+
+        public void logChanges(TreeMap<Long, InternalRequest> requests) {
+            if (requests.isEmpty()) {
+                if (!emptyLogged) {
+                    log.finest("No more pending requests currently.");
+                    emptyLogged = true;
+                }
+            } else {
+                emptyLogged = false;
+                InternalRequest r = requests.firstEntry().getValue();
+                if (lastTimeoutLogged == null || !lastTimeoutLogged.equals(r)) {
+                    lastTimeoutLogged = r;
+                    log.finest("Next operation to possibly timeout will do so at " + r.getTimeoutTime());
+                }
+            }
+        }
+    }
+
+    private final static Logger log = Logger.getLogger(TimeoutHandler.class.getName());
+    private final TreeMap<Long, InternalRequest> requests = new TreeMap<>();
+    private final ChangeLogger changeLogger = new ChangeLogger();
+    private final Clock clock;
+    private boolean run = true;
+    private Runnable timeoutHandler = new Runnable() {
+        @Override
+        public void run() {
+            log.fine("Starting timeout monitor thread");
+            while (true) {
+                performTimeoutHandlerTick();
+                synchronized (clock) {
+                    try{ clock.wait(100); } catch (InterruptedException e) {}
+                    if (!run) break;
+                }
+            }
+            log.fine("Stopped timeout monitor thread");
+        }
+    };
+
+    public TimeoutHandler(Executor executor, Clock clock, AsyncHttpClient<V> client) {
+        super(client);
+        this.clock = clock;
+        executor.execute(timeoutHandler);
+    }
+
+    @Override
+    public void close() {
+        synchronized (clock) {
+            run = false;
+            clock.notifyAll();
+        }
+        synchronized (requests) {
+            for (InternalRequest r : requests.values()) {
+                r.operation.cancel();
+                r.setFailure(new TimeoutException("Timeout handler shutting down. Shutting down all requests monitored."));
+            }
+            requests.clear();
+        }
+    }
+
+    @Override
+    public AsyncOperation<V> execute(HttpRequest r) {
+        AsyncOperation<V> op = super.execute(r);
+        InternalRequest<V> request = new InternalRequest<>(op, clock.getTimeInMillis(), r.getTimeoutMillis());
+        synchronized (requests) {
+            requests.put(request.getTimeoutTime(), request);
+        }
+        return request;
+    }
+
+    void performTimeoutHandlerTick() {
+        synchronized (requests) {
+            removeCompletedRequestsFromTimeoutList();
+            handleTimeoutsAtTime(clock.getTimeInMillis());
+            changeLogger.logChanges(requests);
+        }
+    }
+
+    private void removeCompletedRequestsFromTimeoutList() {
+        while (!requests.isEmpty() && requests.firstEntry().getValue().operation.isDone()) {
+            requests.remove(requests.firstEntry().getKey());
+            log.finest("Removed completed request from operation timeout list.");
+        }
+    }
+
+    private void handleTimeoutsAtTime(long currentTime) {
+        Map<Long, InternalRequest> timeouts = requests.subMap(0l, currentTime + 1);
+        for (InternalRequest r : timeouts.values()) {
+            r.handleTimeout(currentTime);
+            requests.values().remove(r);
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java
new file mode 100644
index 00000000000..72155601ba5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java
new file mode 100644
index 00000000000..ab3816d10f3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriter.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http.writer;
+
+public class HttpWriter {
+    private final StringBuilder builder = new StringBuilder();
+
+    private String title = "Untitled page";
+    enum State { HEADER, BODY, FINALIZED };
+    private State state = State.HEADER;
+
+    public HttpWriter() {
+    }
+
+    public HttpWriter addTitle(String title) {
+        verifyState(State.HEADER);
+        this.title = title;
+        return this;
+    }
+
+    public HttpWriter write(String paragraph) {
+        verifyState(State.BODY);
+        builder.append("    <p>\n")
+               .append("      " + paragraph + "\n")
+               .append("    </p>\n");
+        return this;
+    }
+
+    public HttpWriter writeLink(String name, String link) {
+        verifyState(State.BODY);
+        builder.append("    <a href=\"" + link + "\">" + name + "</a>\n");
+        return this;
+    }
+
+    private void verifyState(State state) {
+        if (this.state == state) return;
+        if (state != State.FINALIZED && this.state == State.FINALIZED) {
+            throw new IllegalStateException("HTTP page already finalized");
+        }
+        if (state == State.HEADER && this.state == State.BODY) {
+            throw new IllegalStateException("Have already started to write body. Cannot alter header");
+        }
+        if (this.state == State.HEADER) {
+            builder.append("<html>\n"
+                    + "  <head>\n"
+                    + "    <title>" + title + "</title>\n"
+                    + "  </head>\n"
+                    + "  <body>\n"
+                    + "    <h1>" + title + "</h1>\n");
+            this.state = State.BODY;
+            if (this.state == state) return;
+        }
+            // If we get here we are in state body and want to get finalized
+        builder.append("  </body>\n"
+                     + "</html>\n");
+        this.state = State.FINALIZED;
+    }
+
+    public String toString() {
+        verifyState(State.FINALIZED);
+        return builder.toString();
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java
new file mode 100644
index 00000000000..b380130d252
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPI.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Interface to implement for backends that want to have a State Rest API.
+ */
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse;
+
+public interface StateRestAPI {
+
+    UnitResponse getState(UnitStateRequest request) throws StateRestApiException;
+
+    SetResponse setUnitState(SetUnitStateRequest request) throws StateRestApiException;
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java
new file mode 100644
index 00000000000..10b629a29e4
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InternalFailure.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InternalFailure extends StateRestApiException {
+
+    public InternalFailure(String description) {
+        super("Internal failure. Should not happen: " + description);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java
new file mode 100644
index 00000000000..86f6dc81926
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidContentException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InvalidContentException extends StateRestApiException {
+
+    public InvalidContentException(String description) {
+        super(description);
+        setHtmlCode(400);
+        setHtmlStatus("Content of HTTP request had invalid data");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java
new file mode 100644
index 00000000000..48700e3f343
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/InvalidOptionValueException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class InvalidOptionValueException extends StateRestApiException {
+
+    public InvalidOptionValueException(String option, String value, String description) {
+        super("Option '" + option + "' have invalid value '" + value + "': " + description);
+        setHtmlCode(400);
+        setHtmlStatus("Option '" + option + "' have invalid value '" + value + "'");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java
new file mode 100644
index 00000000000..4a0cb76f278
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+/**
+ * @author hakon
+ */
+public class MissingResourceException extends StateRestApiException {
+    public MissingResourceException(String resource) {
+        super("Missing resource '" + resource + "'");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java
new file mode 100644
index 00000000000..037d82fa0b0
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class MissingUnitException extends StateRestApiException {
+
+    private static String createMessage(String[] path, int level) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("No such resource '");
+        for (int i=0; i<=level; ++i) {
+            if (i != 0) sb.append('/');
+            sb.append(path[i]);
+        }
+        return sb.append("'.").toString();
+    }
+
+    public MissingUnitException(String[] path, int level) {
+        super(createMessage(path, level));
+        setHtmlCode(404);
+        setHtmlStatus(getMessage());
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java
new file mode 100644
index 00000000000..46e964d77ef
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/NotMasterException.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public abstract class NotMasterException extends StateRestApiException {
+
+    public NotMasterException(String description) {
+        super(description);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java
new file mode 100644
index 00000000000..e9ebccbfb66
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+import java.util.Arrays;
+
+public class OperationNotSupportedForUnitException extends StateRestApiException {
+
+    private static String createMessage(String[] path, String description) {
+        return new StringBuilder()
+                .append(Arrays.toString(path)).append(": ").append(description)
+                .toString();
+    }
+
+    public OperationNotSupportedForUnitException(String path[], String description) {
+        super(createMessage(path, description));
+        setHtmlCode(405);
+        setHtmlStatus("Operation not supported for resource");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java
new file mode 100644
index 00000000000..4a9c4fc60db
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OtherMasterException.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class OtherMasterException extends NotMasterException {
+    private final String masterHost;
+    private final int masterPort;
+
+    public OtherMasterException(String masterHost, int masterPort) {
+        super("Cluster controller not master. Use master at " + masterHost + ":" + masterPort + ".");
+        this.masterHost = masterHost;
+        this.masterPort = masterPort;
+    }
+
+    public String getHost() { return masterHost; }
+    public int getPort() { return masterPort; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java
new file mode 100644
index 00000000000..6509b31c2c9
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/StateRestApiException.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public abstract class StateRestApiException extends Exception {
+    private Integer htmlCode;
+    private String htmlStatus;
+
+    public StateRestApiException(String description) {
+        super(description);
+    }
+
+    /**
+     * If given, this HTML code is set in the response. If not given, a value will
+     * be autogenerated to fit.
+     */
+    public StateRestApiException setHtmlCode(int code) {
+        htmlCode = code;
+        return this;
+    }
+
+    /**
+     * If given, this HTML status string is set in the response. If not given, a value will
+     * be autogenerated to fit.
+     */
+    public StateRestApiException setHtmlStatus(String status) {
+        htmlStatus = status;
+        return this;
+    }
+
+    public Integer getCode() { return htmlCode; }
+    public String getStatus() { return htmlStatus; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
new file mode 100644
index 00000000000..408d4c05092
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+public class UnknownMasterException extends NotMasterException {
+
+    public UnknownMasterException() {
+        super("No known master cluster controller currently exists.");
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java
new file mode 100644
index 00000000000..6925d804cdb
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java
new file mode 100644
index 00000000000..9111172a601
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
new file mode 100644
index 00000000000..972b0c4b82a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState;
+
+import java.util.Map;
+
+public interface SetUnitStateRequest extends UnitRequest {
+
+    Map<String, UnitState> getNewState();
+
+    enum Condition {
+        FORCE(1), // Don't check for any condition before setting unit state
+        SAFE(2); // Only set condition if it is deemed safe (e.g. redundancy is still ok during upgrade)
+
+        public final int value;
+
+        private Condition(int value) {
+            this.value = value;
+        }
+
+        public static Condition fromString(String value) throws InvalidContentException {
+            try {
+                return Condition.valueOf(value.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                throw new InvalidContentException("Invalid value for my enum Condition: " + value);
+            }
+        }
+    }
+    Condition getCondition();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java
new file mode 100644
index 00000000000..4259e837078
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java
@@ -0,0 +1,6 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+public interface UnitRequest {
+    String[] getUnitPath();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java
new file mode 100644
index 00000000000..a2b0210362f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitStateRequest.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+public interface UnitStateRequest extends UnitRequest {
+    public int getRecursiveLevels();
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java
new file mode 100644
index 00000000000..bbd1884e56f
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java
new file mode 100644
index 00000000000..20a1f42794c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/CurrentUnitState.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface CurrentUnitState {
+    public Map<String, UnitState> getStatePerType();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java
new file mode 100644
index 00000000000..9f4ecac84d4
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+/**
+ * The response of a set operation.
+ * @author dybdahl
+ */
+public class SetResponse {
+    private final String reason;
+    private final boolean wasModified;
+
+    public SetResponse(String reason, boolean wasModified) {
+        this.reason = reason;
+        this.wasModified = wasModified;
+    }
+
+    /**
+     * Indicates if data was modified in a set operation.
+     * @return true if modified.
+     */
+    public boolean getWasModified() { return wasModified; }
+
+    /**
+     * Human readable reason.
+     * @return reason as string
+     */
+    public String getReason() { return reason; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java
new file mode 100644
index 00000000000..9415a0f9953
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SubUnitList.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface SubUnitList {
+    /** id to link map. */
+    public Map<String, String> getSubUnitLinks();
+    public Map<String, UnitResponse> getSubUnits();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java
new file mode 100644
index 00000000000..9c5cb6940b5
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitAttributes.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitAttributes {
+    public Map<String, String> getAttributeValues();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java
new file mode 100644
index 00000000000..64b8c88584b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitMetrics {
+    public Map<String, Number> getMetricMap();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java
new file mode 100644
index 00000000000..278c3f6dd9b
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitResponse.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import java.util.Map;
+
+public interface UnitResponse {
+    public UnitAttributes getAttributes();
+    public CurrentUnitState getCurrentState();
+    public Map<String, SubUnitList> getSubUnits();
+    public UnitMetrics getMetrics();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java
new file mode 100644
index 00000000000..a4ecf2b09f9
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitState.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+public interface UnitState {
+    public String getId();
+    public String getReason();
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java
new file mode 100644
index 00000000000..516e48e8047
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.response;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
new file mode 100644
index 00000000000..a25f29f005c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class JsonReader {
+    private static class UnitStateImpl implements UnitState {
+        private final String id;
+        private final String reason;
+
+        public UnitStateImpl(String id, String reason) {
+            this.id = id;
+            this.reason = reason;
+        }
+
+        @Override
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public String getReason() {
+            return reason;
+        }
+    }
+
+    static class SetRequestData {
+        final Map<String, UnitState> stateMap;
+        final SetUnitStateRequest.Condition condition;
+        public SetRequestData(Map<String, UnitState> stateMap, SetUnitStateRequest.Condition condition) {
+            this.stateMap = stateMap;
+            this.condition = condition;
+        }
+    }
+
+    public SetRequestData getStateRequestData(HttpRequest request) throws Exception {
+        JSONObject json = new JSONObject(request.getPostContent().toString());
+
+        final SetUnitStateRequest.Condition condition;
+
+        if (json.has("condition")) {
+            condition = SetUnitStateRequest.Condition.valueOf(json.getString("condition"));
+        } else {
+            condition = SetUnitStateRequest.Condition.FORCE;
+        }
+
+        Map<String, UnitState> stateMap = new HashMap<>();
+        if (!json.has("state")) {
+            throw new InvalidContentException("Set state requests must contain a state object");
+        }
+        Object o = json.get("state");
+        if (!(o instanceof JSONObject)) {
+            throw new InvalidContentException("value of state is not a json object");
+        }
+
+        JSONObject state = (JSONObject) o;
+
+        JSONArray stateTypes = state.names();
+        for (int i=0; i<stateTypes.length(); ++i) {
+            o = stateTypes.get(i);
+            String type = (String) o;
+            o = state.get(type);
+            if (!(o instanceof JSONObject)) {
+                throw new InvalidContentException("value of state->" + type + " is not a json object");
+            }
+            JSONObject userState = (JSONObject) o;
+            String code = "up";
+            if (userState.has("state")) {
+                o = userState.get("state");
+                if (!(o instanceof String)) {
+                    throw new InvalidContentException("value of state->" + type + "->state is not a string");
+                }
+                code = o.toString();
+            }
+            String reason = "";
+            if (userState.has("reason")) {
+                o = userState.get("reason");
+                if (!(o instanceof String)) {
+                    throw new InvalidContentException("value of state->" + type + "->reason is not a string");
+                }
+                reason = o.toString();
+            }
+            stateMap.put(type, new UnitStateImpl(code, reason));
+        }
+        return new SetRequestData(stateMap, condition);
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java
new file mode 100644
index 00000000000..f14637c5fb7
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonWriter.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.Map;
+
+public class JsonWriter {
+    private String pathPrefix = "/";
+
+    public JsonWriter() {
+    }
+
+    public void setDefaultPathPrefix(String defaultPathPrefix) {
+        if (defaultPathPrefix.isEmpty() || defaultPathPrefix.charAt(0) != '/') {
+            throw new IllegalArgumentException("Path prefix must start with a slash");
+        }
+        this.pathPrefix = defaultPathPrefix;
+    }
+
+    public JSONObject createJson(UnitResponse data) throws Exception {
+        JSONObject json = new JSONObject();
+        fillInJson(data, json);
+        return json;
+    }
+
+    public void fillInJson(UnitResponse data, JSONObject json) throws Exception {
+        UnitAttributes attributes = data.getAttributes();
+        if (attributes != null) fillInJson(attributes, json);
+        CurrentUnitState stateData = data.getCurrentState();
+        if (stateData != null) fillInJson(stateData, json);
+        UnitMetrics metrics = data.getMetrics();
+        if (metrics != null) fillInJson(metrics, json);
+        Map<String, SubUnitList> subUnits = data.getSubUnits();
+        if (subUnits != null) fillInJson(subUnits, json);
+    }
+
+    public void fillInJson(CurrentUnitState stateData, JSONObject json) throws Exception {
+        JSONObject stateJson = new JSONObject();
+        json.put("state", stateJson);
+        Map<String, UnitState> state = stateData.getStatePerType();
+        for (Map.Entry<String, UnitState> e : state.entrySet()) {
+            String stateType = e.getKey();
+            UnitState unitState = e.getValue();
+            JSONObject stateTypeJson = new JSONObject()
+                    .put("state", unitState.getId())
+                    .put("reason", unitState.getReason());
+            stateJson.put(stateType, stateTypeJson);
+        }
+    }
+
+    public void fillInJson(UnitMetrics metrics, JSONObject json) throws Exception {
+        JSONObject metricsJson = new JSONObject();
+        for (Map.Entry<String, Number> e : metrics.getMetricMap().entrySet()) {
+            metricsJson.put(e.getKey(), e.getValue());
+        }
+        json.put("metrics", metricsJson);
+    }
+    public void fillInJson(UnitAttributes attributes, JSONObject json) throws Exception {
+        JSONObject attributesJson = new JSONObject();
+        for (Map.Entry<String, String> e : attributes.getAttributeValues().entrySet()) {
+            attributesJson.put(e.getKey(), e.getValue());
+        }
+        json.put("attributes", attributesJson);
+    }
+
+    public void fillInJson(Map<String, SubUnitList> subUnitMap, JSONObject json) throws Exception {
+        for(Map.Entry<String, SubUnitList> e : subUnitMap.entrySet()) {
+            String subUnitType = e.getKey();
+            JSONObject typeJson = new JSONObject();
+            for (Map.Entry<String, String> f : e.getValue().getSubUnitLinks().entrySet()) {
+                JSONObject linkJson = new JSONObject();
+                linkJson.put("link", pathPrefix + "/" + f.getValue());
+                typeJson.put(f.getKey(), linkJson);
+            }
+            for (Map.Entry<String, UnitResponse> f : e.getValue().getSubUnits().entrySet()) {
+                JSONObject subJson = new JSONObject();
+                fillInJson(f.getValue(), subJson);
+                typeJson.put(f.getKey(), subJson);
+            }
+            json.put(subUnitType, typeJson);
+        }
+    }
+
+    public JSONObject createErrorJson(String description) {
+        JSONObject o = new JSONObject();
+        try{
+            o.put("message", description);
+        } catch (JSONException e) {
+            // Can't really do anything if we get an error trying to report an error.
+        }
+        return o;
+    }
+
+    public JSONObject createJson(SetResponse setResponse) throws JSONException {
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("wasModified", setResponse.getWasModified());
+        jsonObject.put("reason", setResponse.getReason());
+        return jsonObject;
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
new file mode 100644
index 00000000000..43208237bbe
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
@@ -0,0 +1,166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.JsonHttpResult;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.StateRestAPI;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.*;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class RestApiHandler implements HttpRequestHandler {
+    private final static Logger log = Logger.getLogger(RestApiHandler.class.getName());
+
+    private final StateRestAPI restApi;
+    private final JsonWriter jsonWriter;
+    private final JsonReader jsonReader = new JsonReader();
+
+    public RestApiHandler(StateRestAPI restApi) {
+        this.restApi = restApi;
+        this.jsonWriter = new JsonWriter();
+    }
+
+    public RestApiHandler setDefaultPathPrefix(String defaultPathPrefix) {
+        jsonWriter.setDefaultPathPrefix(defaultPathPrefix);
+        return this;
+    }
+
+    private static void logRequestException(HttpRequest request, Exception exception, Level level) {
+        String exceptionString = Exceptions.toMessageString(exception);
+        log.log(level, "Failed to process request with URI path " +
+                request.getPath() + ": " + exceptionString);
+    }
+
+    @Override
+    public HttpResult handleRequest(HttpRequest request) {
+        try{
+            final String[] unitPath = createUnitPath(request);
+            if (request.getHttpOperation().equals(HttpRequest.HttpOp.GET)) {
+                final int recursiveLevel = getRecursiveLevel(request);
+                UnitResponse data = restApi.getState(new UnitStateRequest() {
+                    @Override
+                    public int getRecursiveLevels() {
+                        return recursiveLevel;
+                    }
+                    @Override
+                    public String[] getUnitPath() {
+                        return unitPath;
+                    }
+                });
+                return new JsonHttpResult().setJson(jsonWriter.createJson(data));
+            } else {
+                final JsonReader.SetRequestData setRequestdata = jsonReader.getStateRequestData(request);
+                SetResponse setResponse = restApi.setUnitState(new SetUnitStateRequest() {
+                    @Override
+                    public Map<String, UnitState> getNewState() {
+                        return setRequestdata.stateMap;
+                    }
+                    @Override
+                    public String[] getUnitPath() {
+                        return unitPath;
+                    }
+                    @Override
+                    public Condition getCondition() { return setRequestdata.condition; }
+                });
+                return new JsonHttpResult().setJson(jsonWriter.createJson(setResponse));
+            }
+        } catch (OtherMasterException exception) {
+            logRequestException(request, exception, LogLevel.DEBUG);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(307, "Temporary Redirect");
+            result.addHeader("Location", getMasterLocationUrl(request, exception.getHost(), exception.getPort()));
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (UnknownMasterException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(503, "Service Unavailable");
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (StateRestApiException exception) {
+            logRequestException(request, exception, Level.WARNING);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(500, "Failed to process request");
+            if (exception.getStatus() != null) result.setHttpCode(result.getHttpReturnCode(), exception.getStatus());
+            if (exception.getCode() != null) result.setHttpCode(exception.getCode(), result.getHttpReturnCodeDescription());
+            result.setJson(jsonWriter.createErrorJson(exception.getMessage()));
+            return result;
+        } catch (Exception exception) {
+            logRequestException(request, exception, LogLevel.ERROR);
+            JsonHttpResult result = new JsonHttpResult();
+            result.setHttpCode(500, "Failed to process request");
+            result.setJson(jsonWriter.createErrorJson(exception.getClass().getName() + ": " + exception.getMessage()));
+            return result;
+        }
+    }
+
+    private String[] createUnitPath(HttpRequest request) {
+        List<String> path = Arrays.asList(request.getPath().split("/"));
+        return path.subList(3, path.size()).toArray(new String[0]);
+    }
+
+    private int getRecursiveLevel(HttpRequest request) throws StateRestApiException {
+        String val = request.getOption("recursive", "false");
+        if (val.toLowerCase().equals("false")) { return 0; }
+        if (val.toLowerCase().equals("true")) { return Integer.MAX_VALUE; }
+        int level;
+        try{
+            level = Integer.parseInt(val);
+            if (level < 0) throw new NumberFormatException();
+        } catch (NumberFormatException e) {
+            throw new InvalidOptionValueException(
+                    "recursive", val, "Recursive option must be true, false, 0 or a positive integer");
+        }
+        return level;
+    }
+
+    private String getMasterLocationUrl(HttpRequest request, String host, int port) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("http://").append(host).append(':').append(port)
+          .append(request.getPath());
+        if (!request.getUrlOptions().isEmpty()) {
+            boolean first = true;
+            for (HttpRequest.KeyValuePair kvp : request.getUrlOptions()) {
+                sb.append(first ? '?' : '&');
+                first = false;
+                sb.append(httpEscape(kvp.getKey())).append('=').append(httpEscape(kvp.getValue()));
+            }
+        }
+        return sb.toString();
+    }
+
+    private static class Escape {
+        public final String pattern;
+        public final String replaceWith;
+
+        public Escape(String pat, String repl) {
+            this.pattern = pat;
+            this.replaceWith = repl;
+        }
+    }
+    private static List<Escape> escapes = new ArrayList<>();
+    static {
+        escapes.add(new Escape("%", "%25"));
+        escapes.add(new Escape(" ", "%20"));
+        escapes.add(new Escape("\\?", "%3F"));
+        escapes.add(new Escape("=", "%3D"));
+        escapes.add(new Escape("\\&", "%26"));
+    }
+
+    private static String httpEscape(String value) {
+        for(Escape e : escapes) {
+            value = value.replaceAll(e.pattern, e.replaceWith);
+        }
+        return value;
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java
new file mode 100644
index 00000000000..f14520b441c
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.staterestapi.server;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java
new file mode 100644
index 00000000000..f143c3930bf
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClock.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import java.util.logging.Logger;
+
+/**
+ * Unit tests want to fast forward time to avoid waiting for time to pass
+ */
+public class FakeClock extends SettableClock {
+    private static final Logger logger = Logger.getLogger(FakeClock.class.getName());
+    protected long currentTime = 1;
+
+    @Override
+    public long getTimeInMillis() {
+        return currentTime;
+    }
+
+    @Override
+    public void adjust(long adjustment) {
+        synchronized (this) {
+            logger.fine("Adjusting clock, adding " + adjustment + " ms to it.");
+            currentTime += adjustment;
+            notifyAll();
+        }
+    }
+
+    @Override
+    public void set(long newTime) {
+        synchronized (this) {
+            if (newTime < currentTime) {
+                // throw new IllegalArgumentException("Clock attempted to be set to go backwards");
+            }
+            currentTime = newTime;
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java
new file mode 100644
index 00000000000..09ae6d3d510
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/SettableClock.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import com.yahoo.vespa.clustercontroller.utils.util.Clock;
+
+public abstract class SettableClock extends Clock {
+    public abstract void set(long newTime);
+    public abstract void adjust(long adjustment);
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java
new file mode 100644
index 00000000000..032ec5bfe12
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/test/TestTransport.java
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.AsyncHttpClient;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+/**
+ * This class is a utility for unit tests.. You can register HttpRequestHandler instances in it, and then
+ * you can extract an AsyncHttpClient&lt;HttpResult&gt; instance from it, which you can use to talk to the
+ * registered servers. Thus you can do end to end testing of components talking over HTTP without actually
+ * going through HTTP if you are using the HTTP abstraction layer in communication.http package.
+ */
+public class TestTransport {
+    private static final Logger log = Logger.getLogger(TestTransport.class.getName());
+    private static class Handler {
+        HttpRequestHandler handler;
+        String pathPrefix;
+        Handler(HttpRequestHandler h, String prefix) { this.handler = h; this.pathPrefix = prefix; }
+    }
+    private static class Socket {
+        public final String hostname;
+        public final int port;
+
+        Socket(String hostname, int port) {
+            this.hostname = hostname;
+            this.port = port;
+        }
+        @Override
+        public boolean equals(Object o) {
+            if (!(o instanceof Socket)) return false;
+            Socket other = (Socket) o;
+            return (hostname.equals(other.hostname) && port == other.port);
+        }
+        @Override
+        public int hashCode() {
+            return hostname.hashCode() * port;
+        }
+    }
+    private static class Request {
+        public final HttpRequest request;
+        public final AsyncOperationImpl<HttpResult> result;
+
+        Request(HttpRequest r, AsyncOperationImpl<HttpResult> rr) {
+            this.request = r;
+            this.result = rr;
+        }
+    }
+    private final Map<Socket, List<Handler>> handlers = new HashMap<>();
+    private final LinkedList<Request> requests = new LinkedList<>();
+    private final AsyncHttpClient<HttpResult> client = new AsyncHttpClient<HttpResult>() {
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            log.fine("Queueing request " + r);
+            if (r.getHttpOperation() == null) {
+                r = r.clone();
+                r.setHttpOperation(r.getPostContent() == null ? HttpRequest.HttpOp.GET : HttpRequest.HttpOp.POST);
+            }
+            r.verifyComplete();
+            AsyncOperationImpl<HttpResult> op = new AsyncOperationImpl<>(r.toString());
+            synchronized (requests) {
+                requests.addLast(new Request(r, op));
+            }
+            return op;
+        }
+        @Override
+        public void close() { TestTransport.this.close(); }
+    };
+    private boolean running = true;
+    private final Thread workerThread = new Thread() {
+        @Override
+        public void run() {
+            while (running) {
+                synchronized (requests) {
+                    if (requests.isEmpty()) {
+                        try {
+                            requests.wait(100);
+                        } catch (InterruptedException e) { return; }
+                    } else {
+                        Request request = requests.removeFirst();
+                        HttpRequest r = request.request;
+                        log.fine("Processing request " + r);
+                        HttpRequestHandler handler = getHandler(r);
+                        if (handler == null) {
+                            if (log.isLoggable(Level.FINE)) {
+                                log.fine("Failed to find target for request " + r.toString(true));
+                                log.fine("Existing handlers:");
+                                for (Socket socket : handlers.keySet()) {
+                                    log.fine("  Socket " + socket.hostname + ":" + socket.port);
+                                    for (Handler h : handlers.get(socket)) {
+                                        log.fine("    " + h.pathPrefix);
+                                    }
+                                }
+                            }
+                            request.result.setResult(new HttpResult().setHttpCode(
+                                    404, "No such server socket with suitable path prefix found open"));
+                        } else {
+                            try{
+                                request.result.setResult(handler.handleRequest(r));
+                            } catch (Exception e) {
+                                HttpResult result = new HttpResult().setHttpCode(500, e.getMessage());
+                                StringWriter sw = new StringWriter();
+                                e.printStackTrace(new PrintWriter(sw));
+                                result.setContent(sw.toString());
+                                request.result.setResult(result);
+                            }
+                        }
+                        //log.fine("Request " + r.toString(true) + " created result " + request.getSecond().getResult().toString(true));
+                    }
+                }
+            }
+        }
+    };
+
+    public TestTransport() {
+        workerThread.start();
+    }
+
+    public void close() {
+        if (!running) return;
+        running = false;
+        synchronized (requests) { requests.notifyAll(); }
+        try {
+            workerThread.join();
+        } catch (InterruptedException e) {}
+    }
+
+    /** Get an HTTP client that talks to this test transport layer. */
+    public AsyncHttpClient<HttpResult> getClient() { return client; }
+
+    private HttpRequestHandler getHandler(HttpRequest r) {
+        Socket socket = new Socket(r.getHost(), r.getPort());
+        synchronized (this) {
+            List<Handler> handlerList = handlers.get(socket);
+            if (handlerList == null) {
+                log.fine("No socket match");
+                return null;
+            }
+            log.fine("Socket found");
+            for (Handler h : handlers.get(socket)) {
+                if (r.getPath().length() >= h.pathPrefix.length() && r.getPath().substring(0, h.pathPrefix.length()).equals(h.pathPrefix)) {
+                    return h.handler;
+                }
+            }
+            log.fine("No path prefix match");
+        }
+        return null;
+    }
+
+    public void addServer(HttpRequestHandler server, String hostname, int port, String pathPrefix) {
+        Socket socket = new Socket(hostname, port);
+        synchronized (this) {
+            List<Handler> shandlers = handlers.get(socket);
+            if (shandlers == null) {
+                shandlers = new LinkedList<>();
+                handlers.put(socket, shandlers);
+            }
+            shandlers.add(new Handler(server, pathPrefix));
+        }
+    }
+
+    public void removeServer(HttpRequestHandler server, String hostname, int port, String pathPrefix) {
+        Socket socket = new Socket(hostname, port);
+        synchronized (this) {
+            List<Handler> shandlers = handlers.get(socket);
+            if (shandlers == null) return;
+            for (Handler h : shandlers) {
+                if (h.handler == server && h.pathPrefix.equals(pathPrefix)) {
+                    shandlers.remove(h);
+                }
+            }
+        }
+    }
+
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java
new file mode 100644
index 00000000000..d8dccfdd836
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneable.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * To avoid having to catch clone not supported exception everywhere, and create code with lack of
+ * coverage, this class exist to hide the clone not supported exceptions that should never happen.
+ */
+public class CertainlyCloneable<T> implements Cloneable {
+    @Override
+    public Object clone() {
+        try{
+            return callParentClone();
+        } catch (CloneNotSupportedException e) {
+                // Super clone should never throw exception for objects that should certainly be cloneable.
+            throw new Error(e);
+        }
+    }
+
+    protected Object callParentClone() throws CloneNotSupportedException {
+        return super.clone();
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java
new file mode 100644
index 00000000000..fae3983e2d3
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/Clock.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * Wrap access to clock so that we can override it in unit tests
+ */
+public class Clock {
+    public long getTimeInMillis() { return System.currentTimeMillis(); }
+
+    public int getTimeInSecs() { return (int)(getTimeInMillis() / 1000); }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java
new file mode 100644
index 00000000000..d047dcb6bbb
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Metric reporter wrapper to add component name prefix and common dimensions.
+ */
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class ComponentMetricReporter implements MetricReporter {
+    private final MetricReporter impl;
+    private final String prefix;
+    private final Map<String, String> defaultDimensions = new TreeMap<>();
+    private Context defaultContext;
+
+    public ComponentMetricReporter(MetricReporter impl, String prefix) {
+        this.impl = impl;
+        this.prefix = prefix;
+        defaultContext = impl.createContext(defaultDimensions);
+    }
+
+    public ComponentMetricReporter addDimension(String key, String value) {
+        defaultDimensions.put(key, value);
+        defaultContext = impl.createContext(defaultDimensions);
+        return this;
+    }
+
+    public void set(String name, Number value) {
+        impl.set(prefix + name, value, defaultContext);
+    }
+
+    public void add(String name, Number value) {
+        impl.add(prefix + name, value, defaultContext);
+    }
+
+    @Override
+    public void set(String name, Number value, Context context) {
+        impl.set(prefix + name, value, context);
+    }
+
+    @Override
+    public void add(String name, Number value, Context context) {
+        impl.add(prefix + name, value, context);
+    }
+
+    @Override
+    public Context createContext(Map<String, ?> stringMap) {
+        if (stringMap == null) return defaultContext;
+        Map<String, Object> m = new TreeMap<>(stringMap);
+        for(String key : defaultDimensions.keySet()) {
+            if (!m.containsKey(key)) m.put(key, defaultDimensions.get(key));
+        }
+        return impl.createContext(m);
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java
new file mode 100644
index 00000000000..ca9561ac85e
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapper.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * The Jettison json object class has an interface issue where it hides null pointer exceptions
+ * as checked json exceptions. Consequently one has to create catch clauses that code cannot get
+ * into. This class hides those exceptions.
+ *
+ * (Add functions to this wrapper for new functions needing to hide exceptions like this as they are
+ * needed)
+ */
+public class JSONObjectWrapper extends JSONObject {
+
+    @Override
+    public JSONObjectWrapper put(String key, Object value) {
+        try{
+            super.put(key, value);
+            return this;
+        } catch (JSONException e) {
+            throw new NullPointerException(e.getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java
new file mode 100644
index 00000000000..839eebf4bda
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporter.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+/**
+ * Wrapper for current jdisc metrics, such that applications can report metrics without depending on
+ * the whole container. The apputil project will provide an implementation of this interface that
+ * reports on to injected jdisc implementation.
+ */
+public interface MetricReporter {
+    void set(java.lang.String s, java.lang.Number number, Context context);
+
+    void add(java.lang.String s, java.lang.Number number, Context context);
+
+    Context createContext(java.util.Map<java.lang.String,?> dimensions);
+    static interface Context {
+    }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java
new file mode 100644
index 00000000000..ffe6027ce4a
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/NoMetricReporter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import java.util.Map;
+
+public class NoMetricReporter implements MetricReporter {
+    @Override
+    public void set(String s, Number number, Context context) {}
+
+    @Override
+    public void add(String s, Number number, Context context) {}
+
+    @Override
+    public Context createContext(Map<String, ?> stringMap) { return null; }
+}
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java
new file mode 100644
index 00000000000..92aae9f4778
--- /dev/null
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import com.yahoo.osgi.annotation.ExportPackage;