Publish

20 files changed, 1919 insertions, 0 deletions

diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java
new file mode 100644
index 00000000000..855dc2c7263
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/async/AsyncTest.java
@@ -0,0 +1,285 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.async;
+
+import junit.framework.TestCase;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.LinkedList;
+
+public class AsyncTest extends TestCase {
+
+    public void testListeners() {
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        class Listener implements AsyncCallback<String> {
+            boolean called = false;
+            @Override
+            public void done(AsyncOperation<String> op) {
+                called = true;
+            }
+        }
+        Listener l1 = new Listener();
+        Listener l2 = new Listener();
+        Listener l3 = new Listener();
+        Listener l4 = new Listener();
+        op.register(l1);
+        op.register(l2);
+        op.register(l3);
+        op.unregister(l1);
+        op.setResult("foo");
+        op.register(l4);
+            // Listener that is unregistered is not called
+        assertEquals(false, l1.called);
+            // Listener that is registered is called
+        assertEquals(true, l2.called);
+            // Multiple listeners supported
+        assertEquals(true, l3.called);
+            // Listener called directly when registered after result is set
+        assertEquals(true, l4.called);
+    }
+
+    public void testMultipleResultSetters() {
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setResult("foo");
+            op.setResult("bar");
+            assertEquals("foo", op.getResult());
+        }
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setResult("foo");
+            op.setFailure(new Exception("bar"));
+            assertEquals("foo", op.getResult());
+            assertEquals(true, op.isSuccess());
+        }
+        {
+            AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+            op.setFailure(new Exception("bar"));
+            op.setResult("foo");
+            assertNull(op.getResult());
+            assertEquals(false, op.isSuccess());
+            assertEquals("bar", op.getCause().getMessage());
+        }
+    }
+
+    public void testPartialResultOnFailure() {
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        op.setFailure(new Exception("bar"), "foo");
+        assertEquals("foo", op.getResult());
+        assertEquals(false, op.isSuccess());
+        assertEquals("bar", op.getCause().getMessage());
+    }
+
+    public void testListenImpl() {
+        class ListenImpl extends AsyncOperationListenImpl<String> {
+            public ListenImpl(AsyncOperation<String> op) {
+                super(op);
+            }
+        };
+        class Listener implements AsyncCallback<String> {
+            int calls = 0;
+            @Override
+            public void done(AsyncOperation<String> op) {
+                ++calls;
+            }
+        }
+        AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+        ListenImpl impl = new ListenImpl(op);
+        Listener l1 = new Listener();
+        impl.register(l1);
+        impl.notifyListeners();
+        impl.notifyListeners();
+        impl.notifyListeners();
+        assertEquals(1, l1.calls);
+    }
+
+    public void testRedirectedOperation() {
+        {
+            final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test", "desc");
+            AsyncOperation<Integer> deleteRequest = new RedirectedAsyncOperation<String, Integer>(op) {
+                @Override
+                public Integer getResult() {
+                    return Integer.valueOf(op.getResult());
+                }
+            };
+            final LinkedList<Integer> result = new LinkedList<>();
+            deleteRequest.register(new AsyncCallback<Integer>() {
+                @Override
+                public void done(AsyncOperation<Integer> op) {
+                    result.add(op.getResult());
+                }
+            });
+            assertNull(deleteRequest.getProgress());
+            op.setResult("123");
+            assertEquals(true, deleteRequest.isDone());
+            assertEquals(true, deleteRequest.isSuccess());
+            assertEquals(new Integer(123), deleteRequest.getResult());
+            assertEquals("desc", deleteRequest.getDescription());
+            assertEquals("test", deleteRequest.getName());
+            assertEquals(1, result.size());
+            assertEquals(Integer.valueOf(123), result.getFirst());
+            assertEquals(Double.valueOf(1.0), deleteRequest.getProgress());
+
+            // Get some extra coverage
+            deleteRequest.cancel();
+            deleteRequest.isCanceled();
+            deleteRequest.unregister(null);
+        }
+        {
+            final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test", "desc");
+            AsyncOperation<Integer> deleteRequest = new RedirectedAsyncOperation<String, Integer>(op) {
+                @Override
+                public Integer getResult() {
+                    return Integer.valueOf(op.getResult());
+                }
+            };
+            op.setFailure(new Exception("foo"));
+            assertEquals(true, deleteRequest.isDone());
+            assertEquals("foo", deleteRequest.getCause().getMessage());
+            assertEquals(false, deleteRequest.isSuccess());
+            deleteRequest.getProgress();
+        }
+    }
+
+    public void testRedirectOnSuccessOperation() {
+        {
+            final AsyncOperationImpl<Integer> target = new AsyncOperationImpl<>("foo");
+            SuccessfulAsyncCallback<String, Integer> callback = new SuccessfulAsyncCallback<String, Integer>(target) {
+                @Override
+                public void successfullyDone(AsyncOperation<String> source) {
+                    target.setResult(Integer.valueOf(source.getResult()));
+                }
+            };
+            AsyncOperationImpl<String> source = new AsyncOperationImpl<>("source");
+            source.register(callback);
+            source.setResult("5");
+            assertTrue(target.isDone());
+            assertTrue(target.isSuccess());
+            assertEquals(new Integer(5), target.getResult());
+        }
+        {
+            final AsyncOperationImpl<Integer> target = new AsyncOperationImpl<>("foo");
+            SuccessfulAsyncCallback<String, Integer> callback = new SuccessfulAsyncCallback<String, Integer>(target) {
+                @Override
+                public void successfullyDone(AsyncOperation<String> source) {
+                    target.setResult(Integer.valueOf(source.getResult()));
+                }
+            };
+            AsyncOperationImpl<String> source = new AsyncOperationImpl<>("source");
+            source.register(callback);
+            source.setFailure(new RuntimeException("foo"));
+            assertTrue(target.isDone());
+            assertFalse(target.isSuccess());
+            assertEquals("foo", target.getCause().getMessage());
+        }
+    }
+
+    private abstract class StressThread implements Runnable {
+        private final Object monitor;
+        private boolean running = true;
+
+        public StressThread(Object monitor) { this.monitor = monitor; }
+
+        public void stop() {
+            synchronized (monitor) {
+                running = false;
+                monitor.notifyAll();
+            }
+        }
+
+        @Override
+        public void run() {
+            try{ synchronized (monitor) { while (running) {
+                if (hasTask()) {
+                    doTask();
+                } else {
+                    monitor.wait(1000);
+                }
+            } } } catch (Exception e) {}
+        }
+
+        public abstract boolean hasTask();
+        public abstract void doTask();
+    }
+
+    private abstract class AsyncOpStressThread extends StressThread {
+        public AsyncOperationImpl<String> op;
+        public AsyncOpStressThread(Object monitor) { super(monitor); }
+        @Override
+        public boolean hasTask() { return op != null; }
+    }
+
+    private class Completer extends AsyncOpStressThread {
+        public Completer(Object monitor) { super(monitor); }
+        @Override
+        public void doTask() { op.setResult("foo"); op = null; }
+    }
+
+    private class Listener extends AsyncOpStressThread implements AsyncCallback<String> {
+        int counter = 0;
+        int unset = 0;
+        int priorReg = 0;
+        public Listener(Object monitor) { super(monitor); }
+        @Override
+        public void done(AsyncOperation<String> op) {
+            synchronized (this) {
+                if (op.getResult() == null) ++unset;
+                ++counter;
+            }
+        }
+
+        @Override
+        public void doTask() {
+            op.register(this);
+            if (!op.isDone()) ++priorReg;
+            op = null;
+        }
+    }
+
+    public void testStressCompletionAndRegisterToDetectRace() throws Exception {
+        int iterations = 1000;
+        Object monitor = new Object();
+        Completer completer = new Completer(monitor);
+        Listener listener = new Listener(monitor);
+        Thread t1 = new Thread(completer);
+        Thread t2 = new Thread(listener);
+        try{
+            t1.start();
+            t2.start();
+            for (int i=0; i<iterations; ++i) {
+                final AsyncOperationImpl<String> op = new AsyncOperationImpl<>("test");
+                synchronized (monitor) {
+                    completer.op = op;
+                    listener.op = op;
+                    monitor.notifyAll();
+                }
+                while (completer.op != null || listener.op != null) {
+                    try{ Thread.sleep(0); } catch (InterruptedException e) {}
+                }
+            }
+        } finally {
+            completer.stop();
+            listener.stop();
+            t1.join();
+            t2.join();
+        }
+        /*
+        System.out.println("Done with " + iterations + " iterations. "
+                        + "Registered prior " + listener.priorReg + " times. "
+                        + "Unset " + listener.unset + " times. ");
+        // */
+        assertEquals(0, listener.unset);
+        assertEquals(iterations, listener.counter);
+    }
+
+    public void ignoreTestExceptionOnCallback() throws Exception {
+        AsyncOperationImpl<String> impl = new AsyncOperationImpl<>("foo");
+        impl.register(new AsyncCallback<String>() {
+            @Override
+            public void done(AsyncOperation<String> op) {
+                throw new RuntimeException("Foo");
+            }
+        });
+        impl.setResult(null);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java
new file mode 100644
index 00000000000..d8e3983cd32
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/AsyncHttpClientWithBaseTest.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+public class AsyncHttpClientWithBaseTest extends TestCase {
+
+    public void testOverride() {
+        class HttpClient implements AsyncHttpClient<HttpResult> {
+            HttpRequest lastRequest;
+            @Override
+            public AsyncOperation<HttpResult> execute(HttpRequest r) {
+                lastRequest = r;
+                return new AsyncOperationImpl<>("test");
+            }
+            @Override
+            public void close() {
+            }
+        }
+
+        HttpClient client = new HttpClient();
+        AsyncHttpClientWithBase<HttpResult> base = new AsyncHttpClientWithBase<>(client);
+            // No override by default
+        HttpRequest r = new HttpRequest().setPath("/foo").setHost("bar").setPort(50);
+        base.execute(r);
+        assertEquals(client.lastRequest, r);
+            // Base request always set
+        base.setHttpRequestBase(null);
+        base.execute(r);
+        assertEquals(client.lastRequest, r);
+            // Set an override
+        base.setHttpRequestBase(new HttpRequest().setHttpOperation(HttpRequest.HttpOp.DELETE));
+        base.execute(r);
+        assertNotSame(client.lastRequest, r);
+        assertEquals(HttpRequest.HttpOp.DELETE, client.lastRequest.getHttpOperation());
+
+        base.close();
+    }
+
+    public void testClientMustBeSet() {
+        try{
+            new AsyncHttpClientWithBase<HttpResult>(null);
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java
new file mode 100644
index 00000000000..4ef0b4daccc
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/DummyAsyncHttpClient.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+
+public class DummyAsyncHttpClient implements AsyncHttpClient<HttpResult> {
+    HttpResult result;
+    HttpRequest lastRequest;
+
+    public DummyAsyncHttpClient(HttpResult result) {
+        this.result = result;
+    }
+
+    @Override
+    public AsyncOperation<HttpResult> execute(HttpRequest r) {
+        lastRequest = r;
+        AsyncOperationImpl<HttpResult> op = new AsyncOperationImpl<>(r.toString());
+        op.setResult(result);
+        return op;
+    }
+
+    @Override
+    public void close() {
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java
new file mode 100644
index 00000000000..8bd9cfe5dbe
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpRequestTest.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+
+public class HttpRequestTest extends TestCase {
+
+    private HttpRequest createRequest() {
+        return new HttpRequest()
+                .setHost("local")
+                .setPort(20)
+                .addHttpHeader("x-foo", "bar")
+                .setPath("/bah")
+                .setHttpOperation(HttpRequest.HttpOp.PUT)
+                .addUrlOption("urk", "arg")
+                .setTimeout(25);
+    }
+
+    public void testEquality() {
+        assertEquals(createRequest(), createRequest());
+        assertNotSame(createRequest(), createRequest().setHost("localhost"));
+        assertNotSame(createRequest(), createRequest().setPort(40));
+        assertNotSame(createRequest(), createRequest().setPath("/hmm"));
+        assertNotSame(createRequest(), createRequest().addHttpHeader("dsf", "fs"));
+        assertNotSame(createRequest(), createRequest().setHttpOperation(HttpRequest.HttpOp.DELETE));
+    }
+
+    public void testVerifyComplete() {
+        // To be a complete request, an HTTP request must include:
+        //  - A path
+        //  - The HTTP operation type
+        try{
+            new HttpRequest().setPath("/foo").verifyComplete();
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        try{
+            new HttpRequest().setHttpOperation(HttpRequest.HttpOp.GET).verifyComplete();
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        new HttpRequest().setHttpOperation(HttpRequest.HttpOp.GET).setPath("/bar").verifyComplete();
+    }
+
+    public void testMerge() {
+        {
+            HttpRequest base = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            HttpRequest req = new HttpRequest()
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            HttpRequest merged = base.merge(req);
+
+            HttpRequest expected = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addUrlOption("hmm", "arg")
+                    .addHttpHeader("x-foo", "bar");
+            assertEquals(expected, merged);
+        }
+        {
+            HttpRequest base = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.POST)
+                    .addHttpHeader("x-foo", "bar")
+                    .addUrlOption("hmm", "arg");
+            HttpRequest req = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.PUT)
+                    .setPath("/gohere")
+                    .addHttpHeader("Content-Type", "whatevah")
+                    .addUrlOption("tit", "tat")
+                    .setPostContent("foo");
+            HttpRequest merged = base.merge(req);
+
+            HttpRequest expected = new HttpRequest()
+                    .setHttpOperation(HttpRequest.HttpOp.PUT)
+                    .setPath("/gohere")
+                    .addHttpHeader("x-foo", "bar")
+                    .addHttpHeader("Content-Type", "whatevah")
+                    .addUrlOption("hmm", "arg")
+                    .addUrlOption("tit", "tat")
+                    .setPostContent("foo");
+            assertEquals(expected, merged);
+        }
+    }
+
+    public void testNonExistingHeader() {
+        assertEquals("foo", new HttpRequest().getHeader("asd", "foo"));
+        assertEquals("foo", new HttpRequest().addHttpHeader("bar", "foo").getHeader("asd", "foo"));
+    }
+
+    public void testOption() {
+        assertEquals("bar", new HttpRequest().addUrlOption("foo", "bar").getOption("foo", "foo"));
+        assertEquals("foo", new HttpRequest().getOption("asd", "foo"));
+    }
+
+    public void testToString() {
+        assertEquals("GET? http://localhost:8080/",
+                     new HttpRequest()
+                             .setHost("localhost")
+                             .setPort(8080)
+                             .toString(true));
+        assertEquals("POST http://localhost/",
+                new HttpRequest()
+                        .setHttpOperation(HttpRequest.HttpOp.POST)
+                        .setHost("localhost")
+                        .toString(true));
+        assertEquals("GET http://localhost/?foo=bar",
+                new HttpRequest()
+                        .setHttpOperation(HttpRequest.HttpOp.GET)
+                        .addUrlOption("foo", "bar")
+                        .setHost("localhost")
+                        .toString(true));
+    }
+
+    public void testNothingButGetCoverage() {
+        assertEquals(false, new HttpRequest().equals(new Object()));
+        new HttpRequest().getHeaders();
+        new HttpRequest().setUrlOptions(new HttpRequest().getUrlOptions());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java
new file mode 100644
index 00000000000..aa9f019c757
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/HttpResultTest.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+
+public class HttpResultTest extends TestCase {
+
+    public void testSuccess() {
+        assertEquals(false, new HttpResult().setHttpCode(199, "foo").isSuccess());
+        assertEquals(true, new HttpResult().setHttpCode(200, "foo").isSuccess());
+        assertEquals(true, new HttpResult().setHttpCode(299, "foo").isSuccess());
+        assertEquals(false, new HttpResult().setHttpCode(300, "foo").isSuccess());
+    }
+
+    public void testToString() {
+        assertEquals("HTTP 200/OK", new HttpResult().setContent("Foo").toString());
+        assertEquals("HTTP 200/OK\n\nFoo", new HttpResult().setContent("Foo").toString(true));
+        assertEquals("HTTP 200/OK", new HttpResult().toString(true));
+        assertEquals("HTTP 200/OK", new HttpResult().setContent("").toString(true));
+    }
+
+    public void testNothingButGetCoverage() {
+        new HttpResult().getHeaders();
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java
new file mode 100644
index 00000000000..b49b72ef463
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonAsyncHttpClientTest.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonAsyncHttpClientTest extends TestCase {
+
+    public void testJSONInJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+        client.addJsonContentType(true);
+        client.verifyRequestContentAsJson(true);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+        assertTrue(result.isSuccess());
+
+        result.toString();
+        client.close();
+    }
+
+    public void testStringInJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42).toString()));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34).toString());
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+    }
+
+    public void testIllegalJsonIn() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        try {
+            HttpRequest r = new HttpRequest();
+            r.setPostContent("my illegal json");
+
+            client.execute(r);
+            assertTrue(false);
+        } catch (Exception e) {
+
+        }
+    }
+
+    public void testIllegalJSONOut() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent("my illegal json"));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34).toString());
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals("{\"error\":\"Invalid JSON in output: A JSONObject text must begin with '{' at character 1 of my illegal json\",\"output\":\"my illegal json\"}", result.getResult().getJson().toString());
+    }
+
+    public void testEmptyReply() throws Exception {
+        class Client implements AsyncHttpClient<HttpResult> {
+            AsyncOperationImpl<HttpResult> lastOp;
+            @Override
+            public AsyncOperation<HttpResult> execute(HttpRequest r) {
+                return lastOp = new AsyncOperationImpl<>(r.toString());
+            }
+            @Override
+            public void close() {
+            }
+        };
+        Client client = new Client();
+        JsonAsyncHttpClient jsonClient = new JsonAsyncHttpClient(client);
+        AsyncOperation<JsonHttpResult> op = jsonClient.execute(new HttpRequest());
+        client.lastOp.setResult(null);
+        assertNull(op.getResult());
+    }
+
+    public void testNotVerifyingJson() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        JsonAsyncHttpClient client = new JsonAsyncHttpClient(dummy);
+        client.addJsonContentType(true);
+        client.verifyRequestContentAsJson(false);
+
+        HttpRequest r = new HttpRequest();
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        AsyncOperation<JsonHttpResult> result = client.execute(r);
+
+        assertEquals(new JSONObject().put("bar", 42).toString(), result.getResult().getJson().toString());
+        assertTrue(result.isSuccess());
+
+        result.toString();
+        client.close();
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java
new file mode 100644
index 00000000000..cfc11e6dc8d
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/JsonHttpResultTest.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+public class JsonHttpResultTest extends TestCase {
+
+    public void testCopyConstructor() {
+        assertEquals("{}", new JsonHttpResult(new HttpResult()).getJson().toString());
+    }
+
+    public void testOutput() {
+        assertEquals("HTTP 200/OK\n"
+                   + "\n"
+                   + "JSON: {\"foo\": 3}",
+                     new JsonHttpResult(new HttpResult().setContent("{ \"foo\" : 3 }")).toString(true));
+        assertEquals("HTTP 200/OK\n"
+                + "\n"
+                + "{ \"foo\" : }",
+                new JsonHttpResult(new HttpResult().setContent("{ \"foo\" : }")).toString(true));
+    }
+
+    public void testNonJsonOutput() {
+        JsonHttpResult result = new JsonHttpResult();
+        result.setContent("Foo");
+        StringBuilder sb = new StringBuilder();
+        result.printContent(sb);
+        assertEquals("Foo", sb.toString());
+    }
+
+    public void testInvalidJsonOutput() {
+        JsonHttpResult result = new JsonHttpResult();
+        result.setJson(new JSONObject() {
+            @Override
+            public String toString(int indent) throws JSONException {
+                throw new JSONException("Foo");
+            }
+        });
+        StringBuilder sb = new StringBuilder();
+        result.printContent(sb);
+        assertEquals("JSON: {}", sb.toString());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java
new file mode 100644
index 00000000000..cb7ac4d5d31
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/LoggingAsyncHttpClientTest.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class LoggingAsyncHttpClientTest extends TestCase {
+    class HttpClient implements AsyncHttpClient<HttpResult> {
+        AsyncOperationImpl<HttpResult> lastOp;
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            return lastOp = new AsyncOperationImpl<>("test");
+        }
+        @Override
+        public void close() {
+        }
+    }
+
+    public void testWithoutDebugLog() throws Exception {
+        doRequests();
+    }
+
+    public void testWithDebugLog() throws Exception {
+        Logger log = Logger.getLogger(LoggingAsyncHttpClient.class.getName());
+        log.setLevel(Level.FINE);
+        doRequests();
+    }
+
+    private void doRequests() {
+        {
+            HttpClient client = new HttpClient();
+            LoggingAsyncHttpClient<HttpResult> loggingClient = new LoggingAsyncHttpClient<>(client);
+            AsyncOperation<HttpResult> op = loggingClient.execute(new HttpRequest());
+            client.lastOp.setResult(new HttpResult().setContent("foo"));
+            assertEquals("foo", op.getResult().getContent());
+        }
+        {
+            HttpClient client = new HttpClient();
+            LoggingAsyncHttpClient<HttpResult> loggingClient = new LoggingAsyncHttpClient<>(client);
+            AsyncOperation<HttpResult> op = loggingClient.execute(new HttpRequest());
+            client.lastOp.setFailure(new Exception("foo"));
+            assertEquals("foo", op.getCause().getMessage());
+        }
+
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java
new file mode 100644
index 00000000000..2c2e7cfcff3
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/ProxyAsyncHttpClientTest.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class ProxyAsyncHttpClientTest extends TestCase {
+
+    public void testSimple() throws Exception {
+        // Can't really test much here, but verifies that the code runs.
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        ProxyAsyncHttpClient client = new ProxyAsyncHttpClient<>(dummy, "myproxyhost", 1234);
+
+        HttpRequest r = new HttpRequest();
+        r.setPath("/foo");
+        r.setHost("myhost");
+        r.setPort(4567);
+
+        r.setPostContent(new JSONObject().put("foo", 34));
+
+        client.execute(r);
+
+        assertEquals(new HttpRequest().setPath("/myhost:4567/foo")
+                                      .setHost("myproxyhost")
+                                      .setPort(1234)
+                                      .setPostContent(new JSONObject().put("foo", 34)),
+                     dummy.lastRequest);
+    }
+
+    public void testNoAndEmptyPath() throws Exception {
+        DummyAsyncHttpClient dummy = new DummyAsyncHttpClient(
+                new HttpResult().setContent(new JSONObject().put("bar", 42)));
+        ProxyAsyncHttpClient client = new ProxyAsyncHttpClient<>(dummy, "myproxyhost", 1234);
+        try{
+            client.execute(new HttpRequest());
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+            assertTrue(e.getMessage().contains("Host and path must be set prior"));
+        }
+        client.execute(new HttpRequest().setHost("local").setPath(""));
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java
new file mode 100644
index 00000000000..27632047c19
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/RequestQueueTest.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncCallback;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import junit.framework.TestCase;
+
+import java.util.LinkedList;
+
+public class RequestQueueTest extends TestCase {
+    public static class Request {
+        public final HttpRequest request;
+        public final AsyncOperationImpl<HttpResult> result;
+
+        public Request(HttpRequest r, AsyncOperationImpl<HttpResult> rr) {
+            this.request = r;
+            this.result = rr;
+        }
+    }
+
+    public class TestClient implements AsyncHttpClient<HttpResult> {
+        LinkedList<Request> requests = new LinkedList<>();
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            Request p = new Request(r, new AsyncOperationImpl<HttpResult>(r.toString()));
+            synchronized (requests) {
+                requests.addLast(p);
+            }
+            return p.result;
+        }
+        @Override
+        public void close() {}
+    };
+
+    public void testNormalUsage() {
+        TestClient client = new TestClient();
+        RequestQueue<HttpResult> queue = new RequestQueue<>(client, 4);
+        final LinkedList<HttpResult> results = new LinkedList<>();
+        for (int i=0; i<10; ++i) {
+            queue.schedule(new HttpRequest().setPath("/" + i), new AsyncCallback<HttpResult>() {
+                @Override
+                public void done(AsyncOperation<HttpResult> op) {
+                    if (op.isSuccess()) {
+                        results.add(op.getResult());
+                    } else {
+                        results.add(new HttpResult().setHttpCode(500, op.getCause().getMessage()));
+                    }
+                }
+            });
+        }
+        assertEquals(4, client.requests.size());
+        for (int i=0; i<3; ++i) {
+            Request p = client.requests.removeFirst();
+            p.result.setResult(new HttpResult());
+            assertEquals(true, results.getLast().isSuccess());
+        }
+        assertEquals(4, client.requests.size());
+        for (int i=0; i<7; ++i) {
+            Request p = client.requests.removeFirst();
+            p.result.setFailure(new Exception("Fail"));
+            assertEquals(false, results.getLast().isSuccess());
+        }
+        assertEquals(0, client.requests.size());
+        assertEquals(true, queue.empty());
+        assertEquals(10, results.size());
+    }
+
+    public class Waiter implements Runnable {
+        boolean waiting = false;
+        boolean completed = false;
+        RequestQueue<HttpResult> queue;
+        Waiter(RequestQueue<HttpResult> queue) {
+            this.queue = queue;
+        }
+        public void run() {
+            try{
+                waiting = true;
+                queue.waitUntilEmpty();
+            } catch (InterruptedException e) { throw new Error(e); }
+            completed = true;
+        }
+    }
+
+    public void testWaitUntilEmpty() throws Exception {
+        TestClient client = new TestClient();
+        RequestQueue<HttpResult> queue = new RequestQueue<>(client, 4);
+        final LinkedList<HttpResult> result = new LinkedList<>();
+        queue.schedule(new HttpRequest().setPath("/foo"), new AsyncCallback<HttpResult>() {
+            @Override
+            public void done(AsyncOperation<HttpResult> op) {
+                result.add(op.getResult());
+            }
+        });
+        Waiter waiter = new Waiter(queue);
+        Thread thread = new Thread(waiter);
+        thread.start();
+        while (!waiter.waiting) {
+            Thread.sleep(1);
+        }
+        assertEquals(0, result.size());
+        client.requests.getFirst().result.setResult(new HttpResult());
+        while (!waiter.completed) {
+            Thread.sleep(1);
+        }
+        assertEquals(1, result.size());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java
new file mode 100644
index 00000000000..a52c6b0cfaa
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/TimeoutHandlerTest.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperationImpl;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncUtils;
+import com.yahoo.vespa.clustercontroller.utils.test.FakeClock;
+import junit.framework.TestCase;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class TimeoutHandlerTest extends TestCase {
+
+    public class TestClient implements AsyncHttpClient<HttpResult> {
+        AsyncOperationImpl<HttpResult> lastOp;
+        @Override
+        public AsyncOperation<HttpResult> execute(HttpRequest r) {
+            return lastOp = new AsyncOperationImpl<>("test");
+        }
+        @Override
+        public void close() {}
+    };
+
+    private ThreadPoolExecutor executor;
+    private TestClient client;
+    private FakeClock clock;
+    private TimeoutHandler<HttpResult> handler;
+
+    public void setUp() {
+        executor = new ThreadPoolExecutor(10, 100, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1000));
+        clock = new FakeClock();
+        client = new TestClient();
+        handler = new TimeoutHandler<>(executor, clock, client);
+    }
+
+    public void tearDown() {
+        handler.close();
+        executor.shutdown();
+    }
+
+    public void testTimeout() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        assertFalse(op.isDone());
+        clock.adjust(999);
+            // Give it a bit of time for timeout handler to have a chance of timout out prematurely
+        try{ Thread.sleep(1); } catch (InterruptedException e) {}
+        assertFalse(op.isDone());
+        clock.adjust(1);
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertFalse(op.isSuccess());
+        assertTrue(op.getCause().getMessage(), op.getCause().getMessage().contains("Operation timeout"));
+            // After timeout, finishing the original request no longer matter
+        client.lastOp.setResult(new HttpResult());
+        assertFalse(op.isSuccess());
+        assertTrue(op.getCause().getMessage(), op.getCause().getMessage().contains("Operation timeout"));
+    }
+
+    public void testNoTimeout() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(999);
+        assertFalse(op.isDone());
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertTrue(op.isSuccess());
+        assertEquals("foo", op.getResult().getContent());
+    }
+
+    public void testNoTimeoutFailing() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(999);
+        assertFalse(op.isDone());
+        client.lastOp.setFailure(new Exception("foo"));
+        AsyncUtils.waitFor(op);
+        assertTrue(op.isDone());
+        assertFalse(op.isSuccess());
+        assertEquals("foo", op.getCause().getMessage());
+    }
+
+    public void testProvokeCompletedOpPurgeInTimeoutList() {
+        AsyncOperation<HttpResult> op1 = handler.execute(new HttpRequest().setTimeout(1000));
+        AsyncOperationImpl<HttpResult> op1Internal = client.lastOp;
+        clock.adjust(300);
+        AsyncOperation<HttpResult> op2 = handler.execute(new HttpRequest().setTimeout(1000));
+        clock.adjust(300);
+        op1Internal.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op1);
+        clock.adjust(800);
+        AsyncUtils.waitFor(op2);
+        assertEquals(true, op1.isDone());
+        assertEquals(true, op2.isDone());
+        assertEquals(true, op1.isSuccess());
+        assertEquals(false, op2.isSuccess());
+    }
+
+    public void testNothingButGetCoverage() {
+        AsyncOperation<HttpResult> op = handler.execute(new HttpRequest().setTimeout(1000));
+        op.getProgress();
+        op.cancel();
+        assertFalse(op.isCanceled()); // Cancel not currently supported
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+        op.getProgress();
+        op = handler.execute(new HttpRequest().setTimeout(1000));
+        handler.performTimeoutHandlerTick();
+        handler.performTimeoutHandlerTick();
+        client.lastOp.setResult(new HttpResult().setContent("foo"));
+        AsyncUtils.waitFor(op);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java
new file mode 100644
index 00000000000..e8c47554cc2
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/communication/http/writer/HttpWriterTest.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.communication.http.writer;
+
+import junit.framework.TestCase;
+
+public class HttpWriterTest extends TestCase {
+    private static String defaultTitle = "My Title";
+    private static String defaultHeader = "<html>\n"
+                                        + "  <head>\n"
+                                        + "    <title>My Title</title>\n"
+                                        + "  </head>\n"
+                                        + "  <body>\n"
+                                        + "    <h1>My Title</h1>\n";
+    private static String defaultFooter = "  </body>\n"
+                                        + "</html>\n";
+
+
+    public void testStructure() {
+        HttpWriter writer = new HttpWriter();
+        String header = defaultHeader.replace(defaultTitle, "Untitled page");
+        assertEquals(header + defaultFooter, writer.toString());
+    }
+    public void testTitle() {
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle);
+        assertEquals(defaultHeader + defaultFooter, writer.toString());
+    }
+    public void testParagraph() {
+        String paragraph = "This is a paragraph";
+        String paragraph2 = "More text";
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle).write(paragraph).write(paragraph2);
+        String content = "    <p>\n"
+                       + "      " + paragraph + "\n"
+                       + "    </p>\n"
+                       + "    <p>\n"
+                       + "      " + paragraph2 + "\n"
+                       + "    </p>\n";
+        assertEquals(defaultHeader + content + defaultFooter, writer.toString());
+    }
+    public void testLink() {
+        String name = "My link";
+        String link = "/foo/bar?hmm";
+        HttpWriter writer = new HttpWriter().addTitle(defaultTitle).writeLink(name, link);
+        String content = "    <a href=\"" + link + "\">" + name + "</a>\n";
+        assertEquals(defaultHeader + content + defaultFooter, writer.toString());
+    }
+    public void testErrors() {
+        try{
+            HttpWriter writer = new HttpWriter().addTitle(defaultTitle);
+            writer.toString();
+            writer.write("foo");
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+        try{
+            new HttpWriter().write("foo").addTitle("bar");
+            assertTrue(false);
+        } catch (IllegalStateException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java
new file mode 100644
index 00000000000..4f1be08267c
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyBackend.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DummyBackend {
+    public static class Cluster {
+        public String id;
+        public Map<String, Node> nodes = new LinkedHashMap<>();
+
+        public Cluster(String id) { this.id = id; }
+        public Cluster addNode(Node n) { nodes.put(n.id, n); n.clusterId = id; return this; }
+    }
+    public static class Node {
+        public String clusterId;
+        public String id;
+        public int docCount = 0;
+        public String state = "up";
+        public String reason = "";
+        public String group = "mygroup";
+
+        public Node(String id) { this.id = id; }
+
+        public Node setDocCount(int count) { docCount = count; return this; }
+        public Node setState(String state) { this.state = state; return this; }
+    }
+    private Map<String, Cluster> clusters = new LinkedHashMap<>();
+
+    public Map<String, Cluster> getClusters() { return clusters; }
+
+    public DummyBackend addCluster(Cluster c) {
+        clusters.put(c.id, c);
+        return this;
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java
new file mode 100644
index 00000000000..3a5ea520f4a
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java
@@ -0,0 +1,194 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OperationNotSupportedForUnitException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DummyStateApi implements StateRestAPI {
+    private final DummyBackend backend;
+    private Exception induceException;
+
+    public DummyStateApi(DummyBackend backend) {
+        this.backend = backend;
+    }
+
+    public void induceException(StateRestApiException e) {
+        induceException = e;
+    }
+    public void induceException(RuntimeException e) {
+        induceException = e;
+    }
+
+    public class SubUnitListImpl implements SubUnitList {
+        private Map<String, String> links = new LinkedHashMap<>();
+        private Map<String, UnitResponse> values = new LinkedHashMap<>();
+
+        @Override
+        public Map<String, String> getSubUnitLinks() { return links; }
+        @Override
+        public Map<String, UnitResponse> getSubUnits() { return values; }
+
+        public void addUnit(DummyBackend.Cluster cluster, int recursive) {
+            if (recursive == 0) {
+                links.put(cluster.id, cluster.id);
+            } else {
+                values.put(cluster.id, getClusterState(cluster, recursive - 1));
+            }
+        }
+        public void addUnit(DummyBackend.Node node, int recursive) {
+            if (recursive == 0) {
+                String link = node.clusterId + '/' + node.id;
+                links.put(node.id, link);
+            } else {
+                values.put(node.id, getNodeState(node));
+            }
+        }
+    }
+
+    private UnitResponse getClusterList(final int recursive) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() { return null; }
+            @Override
+            public UnitMetrics getMetrics() { return null; }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() {
+                Map<String, SubUnitList> result = new LinkedHashMap<>();
+                SubUnitListImpl subUnits = new SubUnitListImpl();
+                result.put("cluster", subUnits);
+                for (Map.Entry<String, DummyBackend.Cluster> e : backend.getClusters().entrySet()) {
+                    subUnits.addUnit(e.getValue(), recursive);
+                }
+                return result;
+            }
+        };
+    }
+    private UnitResponse getClusterState(final DummyBackend.Cluster cluster, final int recursive) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() { return null; }
+            @Override
+            public UnitMetrics getMetrics() { return null; }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() {
+                Map<String, SubUnitList> result = new LinkedHashMap<>();
+                SubUnitListImpl subUnits = new SubUnitListImpl();
+                result.put("node", subUnits);
+                for (Map.Entry<String, DummyBackend.Node> e : cluster.nodes.entrySet()) {
+                    subUnits.addUnit(e.getValue(), recursive);
+                }
+                return result;
+            }
+        };
+    }
+    private UnitResponse getNodeState(final DummyBackend.Node node) {
+        return new UnitResponse() {
+            @Override
+            public UnitAttributes getAttributes() {
+                return new UnitAttributes() {
+                    @Override
+                    public Map<String, String> getAttributeValues() {
+                        Map<String, String> attrs = new LinkedHashMap<>();
+                        attrs.put("group", node.group);
+                        return attrs;
+                    }
+                };
+            }
+            @Override
+            public Map<String, SubUnitList> getSubUnits() { return null; }
+            @Override
+            public CurrentUnitState getCurrentState() {
+                return new CurrentUnitState() {
+                    @Override
+                    public Map<String, UnitState> getStatePerType() {
+                        Map<String, UnitState> m = new LinkedHashMap<>();
+                        m.put("current", new UnitState() {
+                            @Override
+                            public String getId() { return node.state; }
+                            @Override
+                            public String getReason() { return node.reason; }
+                        });
+                        return m;
+                    }
+                };
+            }
+            @Override
+            public UnitMetrics getMetrics() {
+                return new UnitMetrics() {
+                    @Override
+                    public Map<String, Number> getMetricMap() {
+                        Map<String, Number> m = new LinkedHashMap<>();
+                        m.put("doc-count", node.docCount);
+                        return m;
+                    }
+                };
+            }
+        };
+
+    }
+
+    @Override
+    public UnitResponse getState(UnitStateRequest request) throws StateRestApiException {
+        checkForInducedException();
+        String[] path = request.getUnitPath();
+        if (path.length == 0) {
+            return getClusterList(request.getRecursiveLevels());
+        }
+        final DummyBackend.Cluster c = backend.getClusters().get(path[0]);
+        if (c == null) throw new MissingUnitException(path, 0);
+        if (path.length == 1) {
+            return getClusterState(c, request.getRecursiveLevels());
+        }
+        final DummyBackend.Node n = c.nodes.get(path[1]);
+        if (n == null) throw new MissingUnitException(path, 1);
+        if (path.length == 2) {
+            return getNodeState(n);
+        }
+        throw new MissingUnitException(path, 3);
+    }
+
+    @Override
+    public SetResponse setUnitState(SetUnitStateRequest request) throws StateRestApiException {
+        checkForInducedException();
+        String[] path = request.getUnitPath();
+        if (path.length != 2) {
+            throw new OperationNotSupportedForUnitException(
+                    path, "You can only set states on nodes");
+        }
+        DummyBackend.Node n = null;
+        DummyBackend.Cluster c = backend.getClusters().get(path[0]);
+        if (c != null) {
+            n = c.nodes.get(path[1]);
+        }
+        if (n == null) throw new MissingUnitException(path, 2);
+        Map<String, UnitState> newState = request.getNewState();
+        if (newState.size() != 1 || !newState.containsKey("current")) {
+            throw new InvalidContentException("Only state of type 'current' is allowed to be set.");
+        }
+        n.state = newState.get("current").getId();
+        n.reason = newState.get("current").getReason();
+        return new SetResponse("DummyStateAPI", true);
+    }
+
+    private void checkForInducedException() throws StateRestApiException {
+        if (induceException == null) return;
+        Exception e = induceException;
+        induceException = null;
+        if (e instanceof RuntimeException) {
+            throw (RuntimeException) e;
+        }
+        throw (StateRestApiException) e;
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
new file mode 100644
index 00000000000..a76c86fa4a5
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java
@@ -0,0 +1,470 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.staterestapi;
+
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncOperation;
+import com.yahoo.vespa.clustercontroller.utils.communication.async.AsyncUtils;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult;
+import com.yahoo.vespa.clustercontroller.utils.communication.http.JsonHttpResult;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.*;
+import com.yahoo.vespa.clustercontroller.utils.staterestapi.server.RestApiHandler;
+import com.yahoo.vespa.clustercontroller.utils.test.TestTransport;
+import junit.framework.TestCase;
+import org.codehaus.jettison.json.JSONObject;
+
+public class StateRestAPITest extends TestCase {
+
+    private static void populateDummyBackend(DummyBackend backend) {
+        backend.addCluster(new DummyBackend.Cluster("foo")
+                .addNode(new DummyBackend.Node("1")
+                        .setState("initializing")
+                        .setDocCount(5)
+                )
+                .addNode(new DummyBackend.Node("3")
+                        .setDocCount(8)
+                )
+        ).addCluster(new DummyBackend.Cluster("bar")
+                .addNode(new DummyBackend.Node("2")
+                        .setState("down")
+                )
+        );
+    }
+
+    private DummyStateApi stateApi;
+    private TestTransport testTransport;
+
+    private void setupDummyStateApi() {
+        DummyBackend backend = new DummyBackend();
+        stateApi = new DummyStateApi(backend);
+        populateDummyBackend(backend);
+        testTransport = new TestTransport();
+        RestApiHandler handler = new RestApiHandler(stateApi);
+        handler.setDefaultPathPrefix("/cluster/v2");
+        testTransport.addServer(handler, "host", 80, "/cluster/v2");
+    }
+
+    public void tearDown() {
+        if (testTransport != null) {
+            testTransport.close();
+            testTransport = null;
+        }
+        stateApi = null;
+    }
+
+    private HttpResult execute(HttpRequest request) {
+        request.setHost("host").setPort(80);
+        AsyncOperation<HttpResult> op = testTransport.getClient().execute(request);
+        AsyncUtils.waitFor(op);
+        if (!op.isSuccess()) { // Don't call getCause() unless it fails
+            assertTrue(op.getCause().toString(), op.isSuccess());
+        }
+        assertTrue(op.getResult() != null);
+        return op.getResult();
+    }
+    private JSONObject executeOkJsonRequest(HttpRequest request) {
+        HttpResult result = execute(request);
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        return (JSONObject) result.getContent();
+    }
+
+    public void testTopLevelList() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"cluster\": {\n"
+                        + "  \"foo\": {\"link\": \"\\/cluster\\/v2\\/foo\"},\n"
+                        + "  \"bar\": {\"link\": \"\\/cluster\\/v2\\/bar\"}\n"
+                        + "}}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testClusterState() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"node\": {\n"
+                        + "  \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n"
+                        + "  \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n"
+                        + "}}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testNodeState() throws Exception {
+        setupDummyStateApi();
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3"));
+        assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\n"
+                        + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                        + "  \"state\": {\"current\": {\n"
+                        + "    \"state\": \"up\",\n"
+                        + "    \"reason\": \"\"\n"
+                        + "  }},\n"
+                        + "  \"metrics\": {\"doc-count\": 8}\n"
+                        + "}";
+        assertEquals(expected, ((JSONObject) result.getContent()).toString(2));
+    }
+
+    public void testRecursiveMode() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "true"));
+            String expected =
+                    "{\"cluster\": {\n" +
+                    "  \"foo\": {\"node\": {\n" +
+                    "    \"1\": {\n" +
+                    "      \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "      \"state\": {\"current\": {\n" +
+                    "        \"state\": \"initializing\",\n" +
+                    "        \"reason\": \"\"\n" +
+                    "      }},\n" +
+                    "      \"metrics\": {\"doc-count\": 5}\n" +
+                    "    },\n" +
+                    "    \"3\": {\n" +
+                    "      \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "      \"state\": {\"current\": {\n" +
+                    "        \"state\": \"up\",\n" +
+                    "        \"reason\": \"\"\n" +
+                    "      }},\n" +
+                    "      \"metrics\": {\"doc-count\": 8}\n" +
+                    "    }\n" +
+                    "  }},\n" +
+                    "  \"bar\": {\"node\": {\"2\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"down\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 0}\n" +
+                    "  }}}\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "1"));
+            String expected =
+                    "{\"cluster\": {\n" +
+                    "  \"foo\": {\"node\": {\n" +
+                    "    \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n" +
+                    "    \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n" +
+                    "  }},\n" +
+                    "  \"bar\": {\"node\": {\"2\": {\"link\": \"\\/cluster\\/v2\\/bar\\/2\"}}}\n" +
+                    "}}";
+            // Verify that the actual link does not contain backslash. It's just an artifact of
+            // jettison json output.
+            assertEquals("/cluster/v2/foo/1",
+                         json.getJSONObject("cluster").getJSONObject("foo").getJSONObject("node")
+                                    .getJSONObject("1").getString("link"));
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2/foo").addUrlOption("recursive", "1"));
+            String expected =
+                    "{\"node\": {\n" +
+                    "  \"1\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"initializing\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 5}\n" +
+                    "  },\n" +
+                    "  \"3\": {\n" +
+                    "    \"attributes\": {\"group\": \"mygroup\"},\n" +
+                    "    \"state\": {\"current\": {\n" +
+                    "      \"state\": \"up\",\n" +
+                    "      \"reason\": \"\"\n" +
+                    "    }},\n" +
+                    "    \"metrics\": {\"doc-count\": 8}\n" +
+                    "  }\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(
+                    new HttpRequest().setPath("/cluster/v2/foo").addUrlOption("recursive", "false"));
+            String expected =
+                    "{\"node\": {\n" +
+                    "  \"1\": {\"link\": \"\\/cluster\\/v2\\/foo\\/1\"},\n" +
+                    "  \"3\": {\"link\": \"\\/cluster\\/v2\\/foo\\/3\"}\n" +
+                    "}}";
+            assertEquals(expected, json.toString(2));
+        }
+    }
+
+    public void testSetNodeState() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject()
+                            .put("state", "retired")
+                            .put("reason", "No reason")));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(new HttpRequest().setPath("/cluster/v2/foo/3"));
+            String expected = "{\n"
+                    + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                    + "  \"state\": {\"current\": {\n"
+                    + "    \"state\": \"retired\",\n"
+                    + "    \"reason\": \"No reason\"\n"
+                    + "  }},\n"
+                    + "  \"metrics\": {\"doc-count\": 8}\n"
+                    + "}";
+            assertEquals(json.toString(2), expected, json.toString(2));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject()));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        }
+        {
+            JSONObject json = executeOkJsonRequest(new HttpRequest().setPath("/cluster/v2/foo/3"));
+            String expected = "{\n"
+                    + "  \"attributes\": {\"group\": \"mygroup\"},\n"
+                    + "  \"state\": {\"current\": {\n"
+                    + "    \"state\": \"up\",\n"
+                    + "    \"reason\": \"\"\n"
+                    + "  }},\n"
+                    + "  \"metrics\": {\"doc-count\": 8}\n"
+                    + "}";
+            assertEquals(json.toString(2), expected, json.toString(2));
+        }
+        {
+            JSONObject json = new JSONObject()
+                    .put("state", new JSONObject()
+                            .put("current", new JSONObject()
+                                    .put("state", "retired")
+                                    .put("reason", "No reason")))
+                    .put("condition", "FORCE");
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 200, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            StringBuilder print = new StringBuilder();
+            result.printContent(print);
+            assertEquals(print.toString(),
+                    "JSON: {\n" +
+                            "  \"wasModified\": true,\n" +
+                            "  \"reason\": \"DummyStateAPI\"\n" +
+                            "}");
+        }
+    }
+
+    public void testMissingUnits() throws Exception {
+        setupDummyStateApi();
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/unknown"));
+            assertEquals(result.toString(true), 404, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "No such resource 'unknown'.", result.getHttpReturnCodeDescription());
+            String expected = "{\"message\":\"No such resource 'unknown'.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/1234"));
+            assertEquals(result.toString(true), 404, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "No such resource 'foo/1234'.", result.getHttpReturnCodeDescription());
+            String expected = "{\"message\":\"No such resource 'foo\\/1234'.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testUnknownMaster() throws Exception {
+        setupDummyStateApi();
+        stateApi.induceException(new UnknownMasterException());
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 503, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "Service Unavailable", result.getHttpReturnCodeDescription());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"message\":\"No known master cluster controller currently exists.\"}";
+        assertEquals(expected, result.getContent().toString());
+        assertTrue(result.getHeader("Location") == null);
+    }
+
+    public void testOtherMaster() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new OtherMasterException("example.com", 80));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption(" %=?&", "&?%=").addUrlOption("foo", "bar"));
+            assertEquals(result.toString(true), 307, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Temporary Redirect", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "http://example.com:80/cluster/v2?%20%25%3D%3F%26=%26%3F%25%3D&foo=bar", result.getHeader("Location"));
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Cluster controller not master. Use master at example.com:80.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            stateApi.induceException(new OtherMasterException("example.com", 80));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo"));
+            assertEquals(result.toString(true), 307, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Temporary Redirect", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "http://example.com:80/cluster/v2/foo", result.getHeader("Location"));
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Cluster controller not master. Use master at example.com:80.\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testRuntimeException() throws Exception {
+        setupDummyStateApi();
+        stateApi.induceException(new RuntimeException("Moahaha"));
+        HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+        assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+        assertEquals(result.toString(true), "Failed to process request", result.getHttpReturnCodeDescription());
+        assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+        String expected = "{\"message\":\"java.lang.RuntimeException: Moahaha\"}";
+        assertEquals(expected, result.getContent().toString());
+    }
+
+    public void testClientFailures() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new InvalidContentException("Foo bar"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Foo bar\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            stateApi.induceException(new InvalidOptionValueException("foo", "bar", "Foo can not be bar"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'foo' have invalid value 'bar'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'foo' have invalid value 'bar': Foo can not be bar\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            String path[] = new String[1];
+            path[0] = "foo";
+            stateApi.induceException(new OperationNotSupportedForUnitException(path, "Foo"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 405, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Operation not supported for resource", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"[foo]: Foo\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInternalFailure() throws Exception {
+        setupDummyStateApi();
+        {
+            stateApi.induceException(new InternalFailure("Foo"));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2"));
+            assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Failed to process request", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Internal failure. Should not happen: Foo\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInvalidRecursiveValues() throws Exception {
+        setupDummyStateApi();
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "-5"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'recursive' have invalid value '-5'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'recursive' have invalid value '-5': Recursive option must be true, false, 0 or a positive integer\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+        {
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2").addUrlOption("recursive", "foo"));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Option 'recursive' have invalid value 'foo'", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            String expected = "{\"message\":\"Option 'recursive' have invalid value 'foo': Recursive option must be true, false, 0 or a positive integer\"}";
+            assertEquals(expected, result.getContent().toString());
+        }
+    }
+
+    public void testInvalidJsonInSetStateRequest() throws Exception {
+        setupDummyStateApi();
+        {
+            JSONObject json = new JSONObject();
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("Set state requests must contain a state object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", 5);
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state is not a json object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", 5));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current is not a json object"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject().put("state", 5)));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current->state is not a string"));
+        }
+        {
+            JSONObject json = new JSONObject().put("state", new JSONObject()
+                    .put("current", new JSONObject().put("state", "down").put("reason", 5)));
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 400, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "Content of HTTP request had invalid data", result.getHttpReturnCodeDescription());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            assertTrue(result.toString(true), result.getContent().toString().contains("value of state->current->reason is not a string"));
+        }
+        {
+            JSONObject json = new JSONObject()
+                    .put("state", new JSONObject()
+                            .put("current", new JSONObject()
+                                    .put("state", "retired")
+                                    .put("reason", "No reason")))
+                    .put("condition", "Non existing condition");
+            HttpResult result = execute(new HttpRequest().setPath("/cluster/v2/foo/3").setPostContent(json));
+            assertEquals(result.toString(true), 500, result.getHttpReturnCode());
+            assertEquals(result.toString(true), "application/json", result.getHeader("Content-Type"));
+            StringBuilder print = new StringBuilder();
+            result.printContent(print);
+            assertEquals(print.toString(),
+                    "JSON: {\"message\": \"java.lang.IllegalArgumentException: No enum constant " +
+                            "com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest." +
+                            "Condition.Non existing condition\"}");
+        }
+    }
+
+    public void testInvalidPathPrefix() throws Exception {
+        DummyBackend backend = new DummyBackend();
+        stateApi = new DummyStateApi(backend);
+        populateDummyBackend(backend);
+        testTransport = new TestTransport();
+        RestApiHandler handler = new RestApiHandler(stateApi);
+        try{
+            handler.setDefaultPathPrefix("cluster/v2");
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java
new file mode 100644
index 00000000000..173f6c8704a
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/test/FakeClockTest.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.test;
+
+import junit.framework.TestCase;
+
+public class FakeClockTest extends TestCase {
+
+    public void testSimple() {
+        FakeClock clock = new FakeClock();
+            // Should not start at 0, as that is common not initialized yet value
+        assertTrue(clock.getTimeInMillis() > 0);
+        long start = clock.getTimeInMillis();
+
+        clock.adjust(5);
+        assertEquals(start + 5, clock.getTimeInMillis());
+
+        clock.set(start + 10);
+        assertEquals(start + 10, clock.getTimeInMillis());
+
+        clock.adjust(5);
+        assertEquals(start + 15, clock.getTimeInMillis());
+    }
+
+    /**
+     * @todo This should probably throw exceptions.. However, that doesn't seem to be current behavior. I suspect some tests misuse the clock to reset things to run another test. Should probably be fixed.
+     */
+    public void testTurnTimeBack() {
+        FakeClock clock = new FakeClock();
+        clock.set(1000);
+
+        clock.set(500);
+        assertEquals(500, clock.getTimeInMillis());
+
+        clock.adjust(-100);
+        assertEquals(400, clock.getTimeInMillis());
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java
new file mode 100644
index 00000000000..e2dac056cad
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/CertainlyCloneableTest.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class CertainlyCloneableTest extends TestCase {
+
+    private class Foo extends CertainlyCloneable<Foo> {
+        protected Foo callParentClone() throws CloneNotSupportedException {
+            throw new CloneNotSupportedException("Foo");
+        }
+    }
+
+    public void testSimple() {
+        try{
+            Foo f = new Foo();
+            f.clone();
+            fail("Control should not get here");
+        } catch (Error e) {
+            assertEquals("Foo", e.getCause().getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java
new file mode 100644
index 00000000000..4282b1f2020
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/ClockTest.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class ClockTest extends TestCase {
+
+    public void testNothingButGetCoverage() {
+        long s = new Clock().getTimeInSecs();
+        long ms = new Clock().getTimeInMillis();
+        assertTrue(ms >= 1000 * s);
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java
new file mode 100644
index 00000000000..160ef498023
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/JSONObjectWrapperTest.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+public class JSONObjectWrapperTest extends TestCase {
+
+    public void testExceptionWrapping() {
+        JSONObjectWrapper wrapper = new JSONObjectWrapper();
+        try{
+            wrapper.put(null, "foo");
+        } catch (NullPointerException e) {
+            assertEquals("Null key.", e.getMessage());
+        }
+    }
+}
diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java
new file mode 100644
index 00000000000..1b3ffdc78e6
--- /dev/null
+++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/util/MetricReporterTest.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.clustercontroller.utils.util;
+
+import junit.framework.TestCase;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class MetricReporterTest extends TestCase {
+    static class MetricReporterMock implements MetricReporter {
+        StringBuilder sb = new StringBuilder();
+
+        @Override
+        public void set(String s, Number number, Context context) {
+            sb.append("set(").append(s).append(", ").append(number).append(")\n");
+        }
+
+        @Override
+        public void add(String s, Number number, Context context) {
+            sb.append("add(").append(s).append(", ").append(number).append(")\n");
+        }
+        @Override
+        public Context createContext(Map<String, ?> stringMap) {
+            sb.append("createContext(");
+            for (String s : stringMap.keySet()) {
+                sb.append(" ").append(s).append("=").append(stringMap.get(s));
+            }
+            sb.append(" )\n");
+            return new Context() {};
+        }
+    };
+
+    public void testNoMetricReporter() {
+        NoMetricReporter reporter = new NoMetricReporter();
+        reporter.add("foo", 3, null);
+        reporter.set("foo", 3, null);
+        reporter.createContext(null);
+    }
+
+    public void testPrefix() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        c.add("foo", 2);
+        c.set("bar", 1);
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "add(prefixfoo, 2)\n" +
+                "set(prefixbar, 1)\n", mock.sb.toString());
+
+    }
+
+    public void testWithContext() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        Map<String, Integer> myContext = new TreeMap<>();
+        myContext.put("myvar", 3);
+        c.add("foo", 2, c.createContext(myContext));
+        c.set("bar", 1, c.createContext(myContext));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "createContext( myvar=3 urk=fy )\n" +
+                "add(prefixfoo, 2)\n" +
+                "createContext( myvar=3 urk=fy )\n" +
+                "set(prefixbar, 1)\n", mock.sb.toString());
+    }
+
+    public void testDefaultContext() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        c.add("foo", 2, c.createContext(null));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "add(prefixfoo, 2)\n", mock.sb.toString());
+    }
+
+    public void testContextOverlap() {
+        MetricReporterMock mock = new MetricReporterMock();
+        ComponentMetricReporter c = new ComponentMetricReporter(mock, "prefix");
+        c.addDimension("urk", "fy");
+        Map<String, String> myContext = new TreeMap<>();
+        myContext.put("urk", "yes");
+        c.add("foo", 2, c.createContext(myContext));
+        assertEquals(
+                "createContext( )\n" +
+                "createContext( urk=fy )\n" +
+                "createContext( urk=yes )\n" +
+                "add(prefixfoo, 2)\n", mock.sb.toString());
+    }
+
+}