Add host suspend logic with test and some refactoring.

24 files changed, 943 insertions, 452 deletions

diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml
index f2b31b3afb9..561d42a071c 100644
--- a/node-admin/src/main/application/services.xml
+++ b/node-admin/src/main/application/services.xml
@@ -7,7 +7,7 @@
         <package>com.yahoo.vespa.hosted.node.admin.testapi</package>
       </components>
     </rest-api>
-    <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.NodeAdminScheduler" bundle="node-admin"/>
+    <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater" bundle="node-admin"/>
     <config name='nodeadmin.docker.docker'>
       <caCertPath>/host/docker/certs/ca_cert.pem</caCertPath>
       <clientCertPath>/host/docker/certs/client_cert.pem</clientCertPath>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java
index 6d4873d92bf..f16368907c4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java
@@ -1,155 +1,19 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin;
 
-import com.yahoo.collections.Pair;
 import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.hosted.node.admin.docker.Container;
-import com.yahoo.vespa.hosted.node.admin.docker.Docker;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
 
-import java.io.IOException;
-import java.time.Duration;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
-import java.util.function.Function;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
- * The "most important" class in this module, where the main business logic resides or is driven from.
- *
- * @author stiankri
+ * API for NodeAdmin seen from outside.
+ * @author dybis
  */
-public class NodeAdmin {
-    private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName());
+public interface NodeAdmin {
 
-    private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis();
+     void setState(final List<ContainerNodeSpec> containersToRun);
 
-    private final Docker docker;
-    private final Function<HostName, NodeAgent> nodeAgentFactory;
+     boolean setFreezeAndCheckIfAllFrozen(boolean freeze);
 
-    private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>();
-
-    private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap();
-
-    /**
-     * @param docker interface to docker daemon and docker-related tasks
-     * @param nodeAgentFactory factory for {@link NodeAgent} objects
-     */
-    public NodeAdmin(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) {
-        this.docker = docker;
-        this.nodeAgentFactory = nodeAgentFactory;
-    }
-
-    public void maintainWantedState(final List<ContainerNodeSpec> containersToRun) {
-        final List<Container> existingContainers = docker.getAllManagedContainers();
-
-        synchronizeLocalContainerState(containersToRun, existingContainers);
-
-        garbageCollectDockerImages(containersToRun);
-    }
-
-    private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) {
-        final Set<DockerImage> deletableDockerImages = getDeletableDockerImages(
-                docker.getUnusedDockerImages(), containersToRun);
-        final long currentTime = System.currentTimeMillis();
-        // TODO: This logic should be unit tested.
-        firstTimeEligibleForGC = deletableDockerImages.stream()
-                .collect(Collectors.toMap(
-                        dockerImage -> dockerImage,
-                        dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime)));
-        // Delete images that have been eligible for some time.
-        firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> {
-            if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) {
-                docker.deleteImage(dockerImage);
-            }
-        });
-    }
-
-    // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present.
-    // This is a workaround for Java 8 not having Stream.flatMap(Optional).
-    private static <T> Stream<T> streamOf(Optional<T> opt) {
-        return opt.map(Stream::of)
-                .orElseGet(Stream::empty);
-    }
-
-    static Set<DockerImage> getDeletableDockerImages(
-            final Set<DockerImage> currentlyUnusedDockerImages,
-            final List<ContainerNodeSpec> pendingContainers) {
-        final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream()
-                .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage))
-                .collect(Collectors.toSet());
-        return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture);
-    }
-
-    // Set-difference. Returns minuend minus subtrahend.
-    private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) {
-        final HashSet<T> result = new HashSet<>(minuend);
-        result.removeAll(subtrahend);
-        return result;
-    }
-
-    // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V).
-    // Full outer join means that all elements of both data sources are included in the result,
-    // even when there is no corresponding element (having the same attribute) in the other data set,
-    // in which case the value from the other source will be empty.
-    static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin(
-            final Stream<T> tStream, final Function<T, V> tAttributeExtractor,
-            final Stream<U> uStream, final Function<U, V> uAttributeExtractor) {
-        final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t));
-        final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u));
-        return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream())
-                .distinct()
-                .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key))));
-    }
-
-    void synchronizeLocalContainerState(
-            final List<ContainerNodeSpec> containersToRun,
-            final List<Container> existingContainers) {
-        final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin(
-                containersToRun.stream(), nodeSpec -> nodeSpec.hostname,
-                existingContainers.stream(), container -> container.hostname);
-
-        final Set<HostName> nodeHostNames = containersToRun.stream()
-                .map(spec -> spec.hostname)
-                .collect(Collectors.toSet());
-        final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames);
-        obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).stop());
-
-        nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> {
-            final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst();
-            final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond();
-
-            if (!nodeSpec.isPresent()) {
-                assert existingContainer.isPresent();
-                logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist");
-                return;
-            }
-
-            try {
-                updateAgent(nodeSpec.get());
-            } catch (IOException e) {
-                logger.log(Level.WARNING, "Failed to bring container to desired state", e);
-            }
-        });
-    }
-
-    private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException {
-        final NodeAgent agent;
-        if (nodeAgents.containsKey(nodeSpec.hostname)) {
-            agent = nodeAgents.get(nodeSpec.hostname);
-        } else {
-            agent = nodeAgentFactory.apply(nodeSpec.hostname);
-            nodeAgents.put(nodeSpec.hostname, agent);
-            agent.start();
-        }
-        agent.update();
-    }
+     Set<HostName> getListOfHosts();
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java
new file mode 100644
index 00000000000..68e4337a483
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java
@@ -0,0 +1,171 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Container;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Administers a host (for now only docker hosts) and its nodes (docker containers nodes).
+ *
+ * @author stiankri
+ */
+public class NodeAdminImpl implements NodeAdmin {
+    private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName());
+
+    private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis();
+
+    private final Docker docker;
+    private final Function<HostName, NodeAgent> nodeAgentFactory;
+
+    private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>();
+
+    private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap();
+
+    /**
+     * @param docker interface to docker daemon and docker-related tasks
+     * @param nodeAgentFactory factory for {@link NodeAgent} objects
+     */
+    public NodeAdminImpl(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) {
+        this.docker = docker;
+        this.nodeAgentFactory = nodeAgentFactory;
+    }
+
+    public void setState(final List<ContainerNodeSpec> containersToRun) {
+        final List<Container> existingContainers = docker.getAllManagedContainers();
+
+        synchronizeLocalContainerState(containersToRun, existingContainers);
+
+        garbageCollectDockerImages(containersToRun);
+    }
+
+    public boolean setFreezeAndCheckIfAllFrozen(boolean freeze) {
+        for (NodeAgent nodeAgent : nodeAgents.values()) {
+            nodeAgent.execute(freeze ? NodeAgent.Command.FREEZE : NodeAgent.Command.UNFREEZE);
+        }
+        for (NodeAgent nodeAgent : nodeAgents.values()) {
+            if (freeze && nodeAgent.getState() != NodeAgent.State.FROZEN) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public Set<HostName> getListOfHosts() {
+        return nodeAgents.keySet();
+    }
+
+    private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) {
+        final Set<DockerImage> deletableDockerImages = getDeletableDockerImages(
+                docker.getUnusedDockerImages(), containersToRun);
+        final long currentTime = System.currentTimeMillis();
+        // TODO: This logic should be unit tested.
+        firstTimeEligibleForGC = deletableDockerImages.stream()
+                .collect(Collectors.toMap(
+                        dockerImage -> dockerImage,
+                        dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime)));
+        // Delete images that have been eligible for some time.
+        firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> {
+            if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) {
+                docker.deleteImage(dockerImage);
+            }
+        });
+    }
+
+    // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present.
+    // This is a workaround for Java 8 not having Stream.flatMap(Optional).
+    private static <T> Stream<T> streamOf(Optional<T> opt) {
+        return opt.map(Stream::of)
+                .orElseGet(Stream::empty);
+    }
+
+    static Set<DockerImage> getDeletableDockerImages(
+            final Set<DockerImage> currentlyUnusedDockerImages,
+            final List<ContainerNodeSpec> pendingContainers) {
+        final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream()
+                .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage))
+                .collect(Collectors.toSet());
+        return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture);
+    }
+
+    // Set-difference. Returns minuend minus subtrahend.
+    private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) {
+        final HashSet<T> result = new HashSet<>(minuend);
+        result.removeAll(subtrahend);
+        return result;
+    }
+
+    // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V).
+    // Full outer join means that all elements of both data sources are included in the result,
+    // even when there is no corresponding element (having the same attribute) in the other data set,
+    // in which case the value from the other source will be empty.
+    static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin(
+            final Stream<T> tStream, final Function<T, V> tAttributeExtractor,
+            final Stream<U> uStream, final Function<U, V> uAttributeExtractor) {
+        final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t));
+        final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u));
+        return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream())
+                .distinct()
+                .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key))));
+    }
+
+    void synchronizeLocalContainerState(
+            final List<ContainerNodeSpec> containersToRun,
+            final List<Container> existingContainers) {
+        final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin(
+                containersToRun.stream(), nodeSpec -> nodeSpec.hostname,
+                existingContainers.stream(), container -> container.hostname);
+
+        final Set<HostName> nodeHostNames = containersToRun.stream()
+                .map(spec -> spec.hostname)
+                .collect(Collectors.toSet());
+        final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames);
+        obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).terminate());
+
+        nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> {
+            final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst();
+            final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond();
+
+            if (!nodeSpec.isPresent()) {
+                assert existingContainer.isPresent();
+                logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist");
+                return;
+            }
+
+            try {
+                updateAgent(nodeSpec.get());
+            } catch (IOException e) {
+                logger.log(Level.WARNING, "Failed to bring container to desired state", e);
+            }
+        });
+    }
+
+    private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException {
+        final NodeAgent agent;
+        if (nodeAgents.containsKey(nodeSpec.hostname)) {
+            agent = nodeAgents.get(nodeSpec.hostname);
+        } else {
+            agent = nodeAgentFactory.apply(nodeSpec.hostname);
+            nodeAgents.put(nodeSpec.hostname, agent);
+            agent.start();
+        }
+        agent.execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java
deleted file mode 100644
index 985e20a3ea8..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin;
-
-import com.yahoo.component.AbstractComponent;
-import com.yahoo.log.LogLevel;
-import com.yahoo.nodeadmin.docker.DockerConfig;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.hosted.node.admin.docker.Docker;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerImpl;
-import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
-import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl;
-import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
-import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl;
-
-import javax.annotation.concurrent.GuardedBy;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.function.Function;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-
-/**
- * @author stiankri
- */
-public class NodeAdminScheduler extends AbstractComponent {
-    private static final Logger log = Logger.getLogger(NodeAdminScheduler.class.getName());
-
-    private static final long INITIAL_DELAY_SECONDS = 0;
-    private static final long INTERVAL_IN_SECONDS = 60;
-
-    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-    private final ScheduledFuture<?> scheduledFuture;
-
-    private enum State { WAIT, WORK, STOP }
-
-    private final Object monitor = new Object();
-    @GuardedBy("monitor")
-    private State state = State.WAIT;
-    @GuardedBy("monitor")
-    private List<ContainerNodeSpec> wantedContainerState = null;
-
-    public NodeAdminScheduler(final DockerConfig dockerConfig) {
-        final Docker docker = new DockerImpl(DockerImpl.newDockerClientFromConfig(dockerConfig));
-        final NodeRepository nodeRepository = new NodeRepositoryImpl();
-        final Orchestrator orchestrator = new OrchestratorImpl(OrchestratorImpl.makeOrchestratorHostApiClient());
-        final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) ->
-                new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator);
-        final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory);
-        scheduledFuture = scheduler.scheduleWithFixedDelay(
-                throwableLoggingRunnable(fetchContainersToRunFromNodeRepository(nodeRepository)),
-                INITIAL_DELAY_SECONDS, INTERVAL_IN_SECONDS, SECONDS);
-        new Thread(maintainWantedStateRunnable(nodeAdmin), "Node Admin Scheduler main thread").start();
-    }
-
-    private void notifyWorkToDo(final Runnable codeToExecuteInCriticalSection) {
-        synchronized (monitor) {
-            if (state == State.STOP) {
-                return;
-            }
-            state = State.WORK;
-            codeToExecuteInCriticalSection.run();
-            monitor.notifyAll();
-        }
-    }
-
-    /**
-     * Prevents exceptions from leaking out and suppressing the scheduler from running the task again.
-     */
-    private static Runnable throwableLoggingRunnable(final Runnable task) {
-        return () -> {
-            try {
-                task.run();
-            } catch (Throwable throwable) {
-                log.log(LogLevel.ERROR, "Unhandled exception leaked out to scheduler.", throwable);
-            }
-        };
-    }
-
-    private Runnable fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) {
-        return () -> {
-            // TODO: should the result from the config server contain both active and inactive?
-            final List<ContainerNodeSpec> containersToRun;
-            try {
-                containersToRun = nodeRepository.getContainersToRun();
-            } catch (IOException e) {
-                log.log(Level.WARNING, "Failed fetching container info from node repository", e);
-                return;
-            }
-            setWantedContainerState(containersToRun);
-        };
-    }
-
-    private void setWantedContainerState(final List<ContainerNodeSpec> wantedContainerState) {
-        if (wantedContainerState == null) {
-            throw new IllegalArgumentException("wantedContainerState must not be null");
-        }
-
-        final Runnable codeToExecuteInCriticalSection = () -> this.wantedContainerState = wantedContainerState;
-        notifyWorkToDo(codeToExecuteInCriticalSection);
-    }
-
-    private Runnable maintainWantedStateRunnable(final NodeAdmin nodeAdmin) {
-        return () -> {
-            while (true) {
-                final List<ContainerNodeSpec> containersToRun;
-
-                synchronized (monitor) {
-                    while (state == State.WAIT) {
-                        try {
-                            monitor.wait();
-                        } catch (InterruptedException e) {
-                            // Ignore, properly handled by next loop iteration.
-                        }
-                    }
-                    if (state == State.STOP) {
-                        return;
-                    }
-                    assert state == State.WORK;
-                    assert wantedContainerState != null;
-                    containersToRun = wantedContainerState;
-                    state = State.WAIT;
-                }
-
-                throwableLoggingRunnable(() -> nodeAdmin.maintainWantedState(containersToRun))
-                        .run();
-            }
-        };
-    }
-
-    @Override
-    public void deconstruct() {
-        scheduledFuture.cancel(false);
-        scheduler.shutdown();
-        synchronized (monitor) {
-            state = State.STOP;
-            monitor.notifyAll();
-        }
-    }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java
new file mode 100644
index 00000000000..cb11a356d55
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl;
+import com.yahoo.vespa.hosted.node.admin.util.Environment;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+/**
+ * Pulls information from node repository and forwards containers to run to node admin.
+ *
+ * @author dybis, stiankri
+ */
+public class NodeAdminStateUpdater extends AbstractComponent {
+    private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName());
+
+    private static final long INITIAL_SCHEDULER_DELAY_SECONDS = 0;
+    private static final long INTERVAL_SCHEDULER_IN_SECONDS = 60;
+
+    private static final int HARDCODED_NODEREPOSITORY_PORT = 19071;
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+
+    private static final String ENV_HOSTNAME = "HOSTNAME";
+    private final NodeAdmin nodeAdmin;
+    private boolean isRunningUpdates = true;
+    private final Object monitor = new Object();
+    final Orchestrator orchestrator;
+
+    // For testing.
+    public NodeAdminStateUpdater(
+            final NodeRepository nodeRepository,
+            final NodeAdmin nodeAdmin,
+            long initialSchedulerDelayMillis,
+            long intervalSchedulerInMillis,
+            Orchestrator orchestrator) {
+        scheduler.scheduleWithFixedDelay(
+                ()-> fetchContainersToRunFromNodeRepository(nodeRepository),
+                initialSchedulerDelayMillis,
+                intervalSchedulerInMillis,
+                MILLISECONDS);
+        this.nodeAdmin = nodeAdmin;
+        this.orchestrator = orchestrator;
+    }
+
+    @Inject
+    public NodeAdminStateUpdater(final Docker docker) {
+        // TODO: This logic does not belong here, NodeAdminScheduler should not build NodeAdmin with all
+        // belonging parts.
+        String baseHostName = java.util.Optional.ofNullable(System.getenv(ENV_HOSTNAME))
+                .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset"));
+
+        final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting();
+        if (configServerHosts.isEmpty()) {
+            throw new IllegalStateException("Environment setting for config servers missing or empty.");
+        }
+
+        final NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, HARDCODED_NODEREPOSITORY_PORT, baseHostName);
+
+        orchestrator = OrchestratorImpl.createOrchestratorFromSettings();
+        final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) ->
+                new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator);
+        final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory);
+        scheduler.scheduleWithFixedDelay(()-> fetchContainersToRunFromNodeRepository(nodeRepository),
+                INITIAL_SCHEDULER_DELAY_SECONDS, INTERVAL_SCHEDULER_IN_SECONDS, SECONDS);
+        this.nodeAdmin = nodeAdmin;
+    }
+
+    public Optional<String> setResumeStateAndCheckIfResumed(boolean resume) {
+        synchronized (monitor) {
+            isRunningUpdates = resume;
+        }
+        if (! nodeAdmin.setFreezeAndCheckIfAllFrozen(resume)) {
+            return Optional.of("Not all node agents in correct state yet.");
+        }
+        List<String> hosts = new ArrayList<>();
+        nodeAdmin.getListOfHosts().forEach(host -> hosts.add(host.toString()));
+        if (resume) {
+            return orchestrator.resume(hosts);
+        }
+        return orchestrator.suspend("parenthost", hosts);
+    }
+
+    private void fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) {
+        synchronized (monitor) {
+            if (! isRunningUpdates) {
+                log.log(Level.FINE, "Is frozen, skipping");
+                return;
+            }
+            // TODO: should the result from the config server contain both active and inactive?
+            final List<ContainerNodeSpec> containersToRun;
+            try {
+                containersToRun = nodeRepository.getContainersToRun();
+            } catch (Throwable t) {
+                log.log(Level.WARNING, "Failed fetching container info from node repository", t);
+                return;
+            }
+            if (containersToRun == null) {
+                log.log(Level.WARNING, "Got null from NodeRepo.");
+                return;
+            }
+            try {
+                nodeAdmin.setState(containersToRun);
+            } catch (Throwable t) {
+                log.log(Level.WARNING, "Failed updating node admin: ", t);
+                return;
+            }
+        }
+    }
+
+    @Override
+    public void deconstruct() {
+        scheduler.shutdown();
+        try {
+            if (! scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+                throw new RuntimeException("Did not manage to shutdown scheduler.");
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java
index 54e7ac3e92f..eac1b228378 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java
@@ -2,25 +2,36 @@
 package com.yahoo.vespa.hosted.node.admin;
 
 /**
- * Responsible for management of a single node/container over its lifecycle.
+ * Responsible for management of a single node over its lifecycle.
  * May own its own resources, threads etc. Runs independently, but receives signals
  * on state changes in the environment that may trigger this agent to take actions.
  *
  * @author bakksjo
  */
 public interface NodeAgent {
+
+    enum Command {UPDATE_FROM_NODE_REPO, FREEZE, UNFREEZE}
+    enum State {WAITING, WORKING, DIRTY, FROZEN, TERMINATED}
+
     /**
      * Signals to the agent that it should update the node specification and container state and maintain wanted state.
      *
-     * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after this method call returns.
+     * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after
+     * this method call returns.
      *
      * It is an error to call this method on an instance after stop() has been called.
      */
-    void update();
+    void execute(Command wantedState);
+
+    /**
+     * Returns the state of the agent.
+     */
+    State getState();
 
     /**
      * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously
-     * striving to make the current state equal to the wanted state. The current and wanted state update as part of {@link #update()}.
+     * striving to make the current state equal to the wanted state. The current and wanted state update as part of
+     * {@link #execute(Command)}.
      */
     void start();
 
@@ -29,5 +40,5 @@ public interface NodeAgent {
      * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this
      * method returns, no more actions will be taken by the agent.
      */
-    void stop();
+    void terminate();
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java
index b197b639166..29e38391c65 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java
@@ -19,6 +19,7 @@ import java.util.Arrays;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -41,11 +42,11 @@ public class NodeAgentImpl implements NodeAgent {
 
     private final Thread thread;
 
-    private enum State { WAIT, WORK, STOP }
-
     private final Object monitor = new Object();
     @GuardedBy("monitor")
-    private State state = State.WAIT;
+    private State state = State.WAITING;
+    @GuardedBy("monitor")
+    private State wantedState = State.WAITING;
 
     // The attributes of the last successful noderepo attribute update for this node. Used to avoid redundant calls.
     // Only used internally by maintenance thread; no synchronization necessary.
@@ -55,6 +56,7 @@ public class NodeAgentImpl implements NodeAgent {
 
 
 
+
     /**
      * @param hostName the hostname of the node managed by this agent
      * @param docker interface to docker daemon and docker-related tasks
@@ -75,17 +77,34 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     @Override
-    public void update() {
-        changeStateAndNotify(() -> {
-            this.state = State.WORK;
-        });
+    public void execute(Command command) {
+        synchronized (monitor) {
+            switch (command) {
+                case UPDATE_FROM_NODE_REPO:
+                    wantedState = State.WORKING;
+                    break;
+                case FREEZE:
+                    wantedState = State.FROZEN;
+                    break;
+                case UNFREEZE:
+                    wantedState = State.WORKING;
+                    break;
+            }
+        }
+    }
+
+    @Override
+    public State getState() {
+        synchronized (monitor) {
+            return state;
+        }
     }
 
     @Override
     public void start() {
         logger.log(LogLevel.INFO, logPrefix + "Scheduling start of NodeAgent");
         synchronized (monitor) {
-            if (state == State.STOP) {
+            if (state == State.TERMINATED) {
                 throw new IllegalStateException("Cannot re-start a stopped node agent");
             }
         }
@@ -93,14 +112,15 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     @Override
-    public void stop() {
+    public void terminate() {
         logger.log(LogLevel.INFO, logPrefix + "Scheduling stop of NodeAgent");
-        changeStateAndNotify(() -> {
-            if (state == State.STOP) {
+        synchronized (monitor) {
+            if (state == State.TERMINATED) {
                 throw new IllegalStateException("Cannot stop an already stopped node agent");
             }
-            state = State.STOP;
-        });
+            wantedState = State.TERMINATED;
+        }
+        monitor.notifyAll();
         try {
             thread.join();
         } catch (InterruptedException e) {
@@ -360,38 +380,42 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     private void scheduleWork() {
-        changeStateAndNotify(() -> state = State.WORK);
-    }
-
-    private void changeStateAndNotify(final Runnable stateChanger) {
         synchronized (monitor) {
-            if (state == State.STOP) {
-                return;
+            if (wantedState != State.FROZEN) {
+                wantedState = State.WORKING;
+            } else {
+                logger.log(Level.FINE, "Not scheduling work since in freeze.");
             }
-            stateChanger.run();
-            monitor.notifyAll();
         }
+        monitor.notifyAll();
     }
 
-    private void maintainWantedState() {
+    private void maintainWantedState()  {
         while (true) {
-            synchronized (monitor) {
-                while (state == State.WAIT) {
-                    try {
-                        monitor.wait();
-                    } catch (InterruptedException e) {
-                        // Ignore, properly handled by next loop iteration.
+            try {
+                synchronized (monitor) {
+                    switch (wantedState) {
+                        case WAITING:
+                            state = State.WAITING;
+                            monitor.wait();
+                            continue;
+                        case WORKING:
+                            state = State.WORKING;
+                            break;
+                        case FROZEN:
+                            state = State.FROZEN;
+                            monitor.wait();
+                            break;
+                        case TERMINATED:
+                            return;
                     }
                 }
-                if (state == State.STOP) {
-                    return;
-                }
-                assert state == State.WORK;
-                state = State.WAIT;
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
             }
-
+            // This is WORKING state.
             try {
-                final ContainerNodeSpec nodeSpec = nodeRepository.getContainer(hostname)
+                final ContainerNodeSpec nodeSpec = nodeRepository.getContainerNodeSpec(hostname)
                         .orElseThrow(() ->
                                 new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname)));
                 final Optional<Container> existingContainer = docker.getContainer(hostname);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java
index 4ef7b1b0705..785b4e98efc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.docker;
 
 import com.google.common.base.Joiner;
 import com.google.common.io.CharStreams;
+import com.google.inject.Inject;
 import com.spotify.docker.client.ContainerNotFoundException;
 import com.spotify.docker.client.DefaultDockerClient;
 import com.spotify.docker.client.DockerCertificateException;
@@ -117,14 +118,16 @@ public class DockerImpl implements Docker {
         this.docker = dockerClient;
     }
 
-    public static DockerClient newDockerClientFromConfig(final DockerConfig config) {
-        return DefaultDockerClient.builder().
+    @Inject
+    public DockerImpl(final DockerConfig config) {
+        this(DefaultDockerClient.builder().
                 uri(config.uri()).
                 dockerCertificates(certificates(config)).
                 readTimeoutMillis(TimeUnit.MINUTES.toMillis(30)). // Some operations may take minutes.
-                build();
+                build());
     }
 
+
     private static DockerCertificates certificates(DockerConfig config) {
         try {
             return DockerCertificates.builder()
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java
index 04d68269144..8829c3d4487 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java
@@ -15,7 +15,7 @@ import java.util.Optional;
 public interface NodeRepository {
     List<ContainerNodeSpec> getContainersToRun() throws IOException;
 
-    Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException;
+    Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException;
 
     void updateNodeAttributes(
             HostName hostName,
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
index 87b5ecd3a93..bc159c89781 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
@@ -29,40 +29,18 @@ import java.util.logging.Logger;
  */
 public class NodeRepositoryImpl implements NodeRepository {
     private static final Logger logger = Logger.getLogger(NodeRepositoryImpl.class.getName());
-    private static final int HARDCODED_NODEREPOSITORY_PORT = 19071;
     private static final String NODEREPOSITORY_PATH_PREFIX_NODES_API = "/";
-    private static final String ENV_HOSTNAME = "HOSTNAME";
 
     private JaxRsStrategy<NodeRepositoryApi> nodeRepositoryClient;
     private final String baseHostName;
 
-    public NodeRepositoryImpl() {
-        baseHostName = Optional.ofNullable(System.getenv(ENV_HOSTNAME))
-                .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset"));
-        nodeRepositoryClient = getApi();
-    }
-
-    // For testing
-    NodeRepositoryImpl(String baseHostName, String configserver, int configport) {
-        this.baseHostName = baseHostName;
-        final Set<HostName> configServerHosts = new HashSet<>();
-        configServerHosts.add(new HostName(configserver));
-
-        final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
-        final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
-                configServerHosts, configport, jaxRsClientFactory);
-        nodeRepositoryClient =  jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API);
-    }
-
-    private static JaxRsStrategy<NodeRepositoryApi> getApi() {
-        final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting();
-        if (configServerHosts.isEmpty()) {
-            throw new IllegalStateException("Environment setting for config servers missing or empty.");
-        }
+    public NodeRepositoryImpl(Set<HostName> configServerHosts, int configPort, String baseHostName) {
         final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
         final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
-                configServerHosts, HARDCODED_NODEREPOSITORY_PORT, jaxRsClientFactory);
-        return jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API);
+                configServerHosts, configPort, jaxRsClientFactory);
+        this.nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries(
+                NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API);
+        this.baseHostName = baseHostName;
     }
 
     @Override
@@ -92,11 +70,15 @@ public class NodeRepositoryImpl implements NodeRepository {
     }
 
     @Override
-    public Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException {
-        // TODO Use proper call to node repository
-        return getContainersToRun().stream()
-                .filter(cns -> Objects.equals(hostname, cns.hostname))
-                .findFirst();
+    public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException {
+        final GetNodesResponse response = nodeRepositoryClient.apply(nodeRepositoryApi -> nodeRepositoryApi.getNode(hostName.toString(), true));
+        if (response.nodes.size() == 0) {
+            return Optional.empty();
+        }
+        if (response.nodes.size() != 1) {
+            throw new RuntimeException("Did not get data for one node using hostname=" + hostName.toString() + "\n" + response.toString());
+        }
+        return Optional.of(createContainerNodeSpec(response.nodes.get(0)));
     }
 
     private static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node)
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java
index 36ab89a6718..519cc7ec3d3 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java
@@ -21,6 +21,18 @@ public interface NodeRepositoryApi {
             @QueryParam("parentHost") String hostname,
             @QueryParam("recursive") boolean recursive);
 
+    /**
+     * What is called "host" in NodeRepo is called "node" in node admin in this case.
+     * @param node the node to get data about.
+     * @param recursive set this to true, or you will not get the data you expect.
+     * @return
+     */
+    @GET
+    @Path("/nodes/v2/node/")
+    GetNodesResponse getNode(
+            @QueryParam("hostname") String node,
+            @QueryParam("recursive") boolean recursive);
+
     @PUT
     @Path("/nodes/v2/state/ready/{hostname}")
     // TODO: remove fake return String body; should be void and empty
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java
index 00365b5fc3d..3afa37075a2 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java
@@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.node.admin.orchestrator;
 
 import com.yahoo.vespa.applicationmodel.HostName;
 
+import java.util.List;
+import java.util.Optional;
+
 /**
  * Abstraction for communicating with Orchestrator.
  *
@@ -18,4 +21,14 @@ public interface Orchestrator {
      * Invokes orchestrator resume of a host. Returns whether resume was granted.
      */
     boolean resume(HostName hostName);
+
+    /**
+     * Invokes orchestrator suspend hosts. Returns failure reasons when failing.
+     */
+    Optional<String> suspend(String parentHostName, List<String> hostNames);
+
+    /**
+     * Invokes orchestrator resume of hosts. Returns failure reasons when failing.
+     */
+    Optional<String> resume(List<String> hostName);
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
index 422962bdc58..b4edd0a516f 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
@@ -7,6 +7,9 @@ import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
 import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory;
 import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory;
 import com.yahoo.vespa.orchestrator.restapi.HostApi;
+import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
 import com.yahoo.vespa.applicationmodel.HostName;
 
@@ -14,7 +17,8 @@ import javax.ws.rs.ClientErrorException;
 import javax.ws.rs.NotFoundException;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
-import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -22,6 +26,7 @@ import java.util.logging.Logger;
 /**
  * @author stiankri
  * @author bakksjo
+ * @author dybis
  */
 public class OrchestratorImpl implements Orchestrator {
     private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName());
@@ -32,19 +37,21 @@ public class OrchestratorImpl implements Orchestrator {
     private static final String ORCHESTRATOR_PATH_PREFIX_HOST_API
             = ORCHESTRATOR_PATH_PREFIX + HostApi.PATH_PREFIX;
 
-    // We use this to allow client code to treat resume() calls as idempotent and cheap,
-    // but we actually filter out redundant resume calls to orchestrator.
-    private final Set<HostName> resumedHosts = new HashSet<>();
+    private static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API
+            = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX;
+
 
     private final JaxRsStrategy<HostApi> hostApiClient;
+    private final JaxRsStrategy<HostSuspensionApi> hostSuspensionClient;
+
 
-    public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient) {
+    public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient, JaxRsStrategy<HostSuspensionApi> hostSuspensionClient) {
         this.hostApiClient = hostApiClient;
+        this.hostSuspensionClient = hostSuspensionClient;
     }
 
     @Override
     public boolean suspend(final HostName hostName) {
-        resumedHosts.remove(hostName);
         try {
             return hostApiClient.apply(api -> {
                 final UpdateHostResponse response = api.suspend(hostName.s());
@@ -64,19 +71,33 @@ public class OrchestratorImpl implements Orchestrator {
     }
 
     @Override
-    public boolean resume(final HostName hostName) {
-        if (resumedHosts.contains(hostName)) {
-            return true;
+    public Optional<String> suspend(String parentHostName, List<String> hostNames) {
+        try {
+            return hostSuspensionClient.apply(hostSuspensionClient -> {
+                BatchHostSuspendRequest request = new BatchHostSuspendRequest(parentHostName, hostNames);
+                final BatchOperationResult result = hostSuspensionClient.suspendAll(request);
+                return result.getFailureReason();
+            });
+        } catch (ClientErrorException e) {
+            if (e instanceof NotFoundException || e.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) {
+                // Orchestrator doesn't care about this node, so don't let that stop us.
+                return Optional.empty();
+            }
+            logger.log(Level.INFO, "Orchestrator rejected suspend request for host " + parentHostName, e);
+            return Optional.of(e.getLocalizedMessage());
+        } catch (IOException e) {
+            logger.log(Level.WARNING, "Unable to communicate with orchestrator", e);
+            return Optional.of("Unable to communicate with orchestrator" + e.getMessage());
         }
+    }
 
+    @Override
+    public boolean resume(final HostName hostName) {
         try {
             final boolean resumeSucceeded = hostApiClient.apply(api -> {
                 final UpdateHostResponse response = api.resume(hostName.s());
                 return response.reason() == null;
             });
-            if (resumeSucceeded) {
-                resumedHosts.add(hostName);
-            }
             return resumeSucceeded;
         } catch (ClientErrorException e) {
             logger.log(Level.INFO, "Orchestrator rejected resume request for host " + hostName, e);
@@ -87,7 +108,17 @@ public class OrchestratorImpl implements Orchestrator {
         }
     }
 
-    public static JaxRsStrategy<HostApi> makeOrchestratorHostApiClient() {
+    @Override
+    public Optional<String> resume(List<String> hostNames) {
+        for (String host : hostNames) {
+            if (! resume(new HostName(host))) {
+                return Optional.of("Could not resume " + host);
+            }
+        }
+        return Optional.empty();
+    }
+
+    public static OrchestratorImpl createOrchestratorFromSettings() {
         final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting();
         if (configServerHosts.isEmpty()) {
             throw new IllegalStateException("Emnvironment setting for config servers missing or empty.");
@@ -95,7 +126,8 @@ public class OrchestratorImpl implements Orchestrator {
         final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
         final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
                 configServerHosts, HARDCODED_ORCHESTRATOR_PORT, jaxRsClientFactory);
-        return jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API);
+        JaxRsStrategy<HostApi> hostApi = jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API);
+        JaxRsStrategy<HostSuspensionApi> suspendApi = jaxRsStrategyFactory.apiWithRetries(HostSuspensionApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API);
+        return new OrchestratorImpl(hostApi, suspendApi);
     }
-
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java
deleted file mode 100644
index 9c2af68d56a..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java
+++ /dev/null
@@ -1,15 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.restapi;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-
-/**
- * @author stiankri
- */
-@Path("")
-public interface NodeAdminRestAPI {
-    @GET
-    @Path("/update")
-    public String update();
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java
new file mode 100644
index 00000000000..0296f48ee95
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java
@@ -0,0 +1,80 @@
+package com.yahoo.vespa.hosted.node.admin.restapi;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+
+import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+
+import static com.yahoo.jdisc.http.HttpRequest.Method.PUT;
+
+/**
+ * @autor dybis
+ */
+public class RestApiHandler extends LoggingRequestHandler{
+
+    private final NodeAdminStateUpdater refresher;
+
+    public RestApiHandler(Executor executor, AccessLog accessLog, NodeAdminStateUpdater refresher) {
+        super(executor, accessLog);
+        this.refresher = refresher;
+    }
+
+    private static class SimpleResponse extends HttpResponse {
+
+        private final String jsonMessage;
+
+        public SimpleResponse(int code, String message) {
+            super(code);
+            // TODO: Use some library to build json as this easily fails
+            this.jsonMessage = "{ \"jsonMessage\":\"" + message + "\"}";
+        }
+
+        @Override
+        public String getContentType() {
+            return MediaType.APPLICATION_JSON;
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            outputStream.write(jsonMessage.getBytes(StandardCharsets.UTF_8.name()));
+        }
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        // TODO Add get call for getting state.
+        if (request.getMethod() != PUT) {
+            return new SimpleResponse(400, "Only PUT is implemented.");
+        }
+        return handlePut(request);
+    }
+
+    private HttpResponse handlePut(HttpRequest request) {
+        String path = request.getUri().getPath();
+        // Check paths to disallow illegal state changes
+        if (path.endsWith("resume")) {
+            final Optional<String> resumed = refresher.setResumeStateAndCheckIfResumed(false);
+            if (resumed.isPresent()) {
+                return new SimpleResponse(400, resumed.get());
+            }
+            return new SimpleResponse(200, "ok.");
+        }
+        if (path.endsWith("suspend")) {
+            Optional<String> resumed = refresher.setResumeStateAndCheckIfResumed(true);
+            if (resumed.isPresent()) {
+                return new SimpleResponse(423, resumed.get());
+            }
+            return new SimpleResponse(200, "ok");
+        }
+        return new SimpleResponse(400, "unknown path" + path);
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java
deleted file mode 100644
index 65da2004854..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.testapi;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-
-/**
- * Resource for use in integration test, will be deleted soon.
- * @author tonytv
- */
-@Path("ping")
-public class PingResource {
-    @GET
-    @Produces(MediaType.TEXT_PLAIN)
-    public String ping() {
-        return "pong";
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java
index a4843a0753e..f3bfeb47e9f 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java
@@ -34,7 +34,7 @@ import static org.mockito.Mockito.when;
 /**
  * @author bakksjo
  */
-public class NodeAdminTest {
+public class NodeAdminImplTest {
     private static final Optional<Double> MIN_CPU_CORES = Optional.of(1.0);
     private static final Optional<Double> MIN_MAIN_MEMORY_AVAILABLE_GB = Optional.of(1.0);
     private static final Optional<Double> MIN_DISK_AVAILABLE_GB = Optional.of(1.0);
@@ -47,7 +47,7 @@ public class NodeAdminTest {
         final Docker docker = mock(Docker.class);
         final Function<HostName, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class);
 
-        final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory);
+        final NodeAdminImpl nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory);
 
         final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class);
         final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class);
@@ -76,31 +76,31 @@ public class NodeAdminTest {
         nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
         inOrder.verify(nodeAgentFactory).apply(hostName);
         inOrder.verify(nodeAgent1).start();
-        inOrder.verify(nodeAgent1).update();
-        inOrder.verify(nodeAgent1, never()).stop();
+        inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+        inOrder.verify(nodeAgent1, never()).terminate();
 
         nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
         inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
         inOrder.verify(nodeAgent1, never()).start();
-        inOrder.verify(nodeAgent1).update();
-        inOrder.verify(nodeAgent1, never()).stop();
+        inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+        inOrder.verify(nodeAgent1, never()).terminate();
 
         nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), asList(existingContainer));
         inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
-        inOrder.verify(nodeAgent1, never()).update();
-        verify(nodeAgent1).stop();
+        inOrder.verify(nodeAgent1, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+        verify(nodeAgent1).terminate();
 
         nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
         inOrder.verify(nodeAgentFactory).apply(hostName);
         inOrder.verify(nodeAgent2).start();
-        inOrder.verify(nodeAgent2).update();
-        inOrder.verify(nodeAgent2, never()).stop();
+        inOrder.verify(nodeAgent2).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+        inOrder.verify(nodeAgent2, never()).terminate();
 
         nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), Collections.emptyList());
         inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
         inOrder.verify(nodeAgent2, never()).start();
-        inOrder.verify(nodeAgent2, never()).update();
-        inOrder.verify(nodeAgent2).stop();
+        inOrder.verify(nodeAgent2, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO);
+        inOrder.verify(nodeAgent2).terminate();
 
         verifyNoMoreInteractions(nodeAgent1);
         verifyNoMoreInteractions(nodeAgent2);
@@ -116,7 +116,7 @@ public class NodeAdminTest {
         final Set<DockerImage> currentlyUnusedImages = Collections.emptySet();
         final List<ContainerNodeSpec> pendingContainers = Collections.emptyList();
 
-        final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+        final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
 
         assertThat(deletableImages, is(Collections.emptySet()));
     }
@@ -127,7 +127,7 @@ public class NodeAdminTest {
                 .collect(Collectors.toSet());
         final List<ContainerNodeSpec> pendingContainers = Collections.emptyList();
 
-        final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+        final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
 
         final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3)
                 .collect(Collectors.toSet());
@@ -139,10 +139,10 @@ public class NodeAdminTest {
         final Set<DockerImage> currentlyUnusedImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3)
                 .collect(Collectors.toSet());
         final List<ContainerNodeSpec> pendingContainers = Stream.of(IMAGE_2, IMAGE_4)
-                .map(NodeAdminTest::newNodeSpec)
+                .map(NodeAdminImplTest::newNodeSpec)
                 .collect(Collectors.toList());
 
-        final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+        final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
 
         final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_3)
                 .collect(Collectors.toSet());
@@ -168,7 +168,7 @@ public class NodeAdminTest {
                 newPair(null, 21)));
 
         assertThat(
-                NodeAdmin.fullOuterJoin(
+                NodeAdminImpl.fullOuterJoin(
                         strings.stream(), string -> string,
                         integers.stream(), String::valueOf)
                         .collect(Collectors.toSet()),
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java
new file mode 100644
index 00000000000..6a5cabd0972
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java
@@ -0,0 +1,79 @@
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.prelude.semantics.RuleBaseException;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import org.junit.Test;
+import org.mockito.stubbing.Answer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.junit.MatcherAssert.assertThat;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Basic test of ActiveContainersRefresherTest
+ * @author dybis
+ */
+public class NodeAdminStateUpdaterTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testExceptionIsCaughtAndDataIsPassedAndFreeze() throws Exception {
+        NodeRepository nodeRepository = mock(NodeRepository.class);
+        NodeAdmin nodeAdmin = mock(NodeAdmin.class);
+        final List<ContainerNodeSpec> accumulatedArgumentList = Collections.synchronizedList(new ArrayList<>());
+        final CountDownLatch latch = new CountDownLatch(5);
+        doAnswer((Answer<Object>) invocation -> {
+            List<ContainerNodeSpec> containersToRunInArgument = (List<ContainerNodeSpec>) invocation.getArguments()[0];
+            containersToRunInArgument.forEach(element -> accumulatedArgumentList.add(element));
+            latch.countDown();
+            if (accumulatedArgumentList.size() == 2) {
+                throw new RuleBaseException("This exception is expected, and should show up in the log.");
+            }
+            return null;
+        }).when(nodeAdmin).setState(anyList());
+
+        final List<ContainerNodeSpec> containersToRun = new ArrayList<>();
+        containersToRun.add(createSample());
+
+        when(nodeRepository.getContainersToRun()).thenReturn(containersToRun);
+        NodeAdminStateUpdater refresher = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, 1, 1, null /* orchestrator*/);
+        latch.await();
+        int numberOfElements = accumulatedArgumentList.size();
+        assertThat(refresher.setResumeStateAndCheckIfResumed(false), is(Optional.of("Not all node agents in correct state yet.")));
+        assertTrue(numberOfElements > 4);
+        assertThat(accumulatedArgumentList.get(0), is(createSample()));
+        Thread.sleep(2);
+        assertThat(accumulatedArgumentList.size(), is(numberOfElements));
+        assertThat(refresher.setResumeStateAndCheckIfResumed(true), is(Optional.of("Not all node agents in correct state yet.")));
+        while (accumulatedArgumentList.size() == numberOfElements) {
+            Thread.sleep(1);
+        }
+        refresher.deconstruct();
+    }
+
+    private ContainerNodeSpec createSample() {
+        return new ContainerNodeSpec(
+                new HostName("hostname"),
+                Optional.empty(),
+                new ContainerName("containername"),
+                NodeState.ACTIVE,
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty());
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java
new file mode 100644
index 00000000000..8500826d005
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java
@@ -0,0 +1,46 @@
+package com.yahoo.vespa.hosted.node.admin.integrationTests;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.NodeAdmin;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Mock with some simple logic
+ * @autor dybis
+ */
+public class NodeAdminMock implements NodeAdmin {
+
+    Set<HostName> hostnames = new HashSet<>();
+
+    boolean freezeSetState = false;
+    public AtomicBoolean frozen = new AtomicBoolean(false);
+
+    // We make it threadsafe as the test have its own peeking thread.
+    private Object monitor = new Object();
+
+    @Override
+    public void setState(List<ContainerNodeSpec> containersToRun) {
+        synchronized (monitor) {
+            hostnames.clear();
+            containersToRun.forEach(container -> hostnames.add(container.hostname));
+        }
+    }
+
+    @Override
+    public boolean setFreezeAndCheckIfAllFrozen(boolean freeze) {
+        freezeSetState = freeze;
+        return frozen.get();
+    }
+
+    @Override
+    public Set<HostName> getListOfHosts() {
+        synchronized (monitor) {
+            return hostnames;
+        }
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
new file mode 100644
index 00000000000..a5233a07ce0
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
@@ -0,0 +1,40 @@
+package com.yahoo.vespa.hosted.node.admin.integrationTests;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Mock with some simple logic
+ * @autor dybis
+ */
+public class NodeRepoMock implements NodeRepository {
+
+    public List<ContainerNodeSpec> containerNodeSpecs = new ArrayList<>();
+
+    @Override
+    public List<ContainerNodeSpec> getContainersToRun() throws IOException {
+        return containerNodeSpecs;
+    }
+
+    @Override
+    public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException {
+        return null;
+    }
+
+    @Override
+    public void updateNodeAttributes(HostName hostName, long restartGeneration, DockerImage dockerImage, String containerVespaVersion) throws IOException {
+
+    }
+
+    @Override
+    public void markAsReady(HostName hostName) throws IOException {
+
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
new file mode 100644
index 00000000000..cdd1dc10bf3
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java
@@ -0,0 +1,36 @@
+package com.yahoo.vespa.hosted.node.admin.integrationTests;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Mock with some simple logic
+ * @autor dybis
+ */
+public class OrchestratorMock implements Orchestrator {
+
+    public Optional<String> suspendReturnValue = Optional.empty();
+
+    @Override
+    public boolean suspend(HostName hostName) {
+        return false;
+    }
+
+    @Override
+    public boolean resume(HostName hostName) {
+        return false;
+    }
+
+    @Override
+    public Optional<String> suspend(String parentHostName, List<String> hostNames) {
+        return suspendReturnValue;
+    }
+
+    @Override
+    public Optional<String> resume(List<String> hostName) {
+        return Optional.empty();
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java
new file mode 100644
index 00000000000..a8cc5e87bf5
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java
@@ -0,0 +1,75 @@
+package com.yahoo.vespa.hosted.node.admin.integrationTests;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Scenario test for NodeAdminStateUpdater.
+ * @author dybis
+ */
+public class ResumeTest {
+    @Test
+    public void test() throws InterruptedException {
+        NodeRepoMock nodeRepositoryMock = new NodeRepoMock();
+        NodeAdminMock nodeAdminMock = new NodeAdminMock();
+        OrchestratorMock orchestratorMock = new OrchestratorMock();
+
+        nodeRepositoryMock.containerNodeSpecs.add(new ContainerNodeSpec(
+                new HostName("hostname"),
+                Optional.of(new DockerImage("dockerimage")),
+                new ContainerName("containe"),
+                NodeState.ACTIVE,
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty()));
+
+        NodeAdminStateUpdater updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdminMock, 1, 1, orchestratorMock);
+        // Wait for node admin to be notified with node repo state
+        while (nodeAdminMock.getListOfHosts().size() == 0) {
+            Thread.sleep(1);
+        }
+
+        // Make node admin and orchestrator block suspend
+        orchestratorMock.suspendReturnValue = Optional.of("orch reject suspend");
+        nodeAdminMock.frozen.set(false);
+        assertThat(updater.setResumeStateAndCheckIfResumed(false), is(Optional.of("Not all node agents in correct state yet.")));
+
+        // Now, change data in node repo, should not propagate.
+        nodeRepositoryMock.containerNodeSpecs.clear();
+
+        // Set node admin not blocking for suspend
+        nodeAdminMock.frozen.set(true);
+        assertThat(updater.setResumeStateAndCheckIfResumed(false), is(Optional.of("orch reject suspend")));
+
+        // Make orchestrator allow suspend
+        orchestratorMock.suspendReturnValue = Optional.empty();
+        assertThat(updater.setResumeStateAndCheckIfResumed(false), is(Optional.empty()));
+
+        // Now suspended, new node repo state should have not propagated to node admin
+        Thread.sleep(2);
+        assertThat(nodeAdminMock.getListOfHosts().size(), is(1));
+
+        // Now resume
+        assertThat(updater.setResumeStateAndCheckIfResumed(true), is(Optional.empty()));
+
+        // Now node repo state should propagate to node admin again
+        while (nodeAdminMock.getListOfHosts().size() != 0) {
+            Thread.sleep(1);
+        }
+
+        updater.deconstruct();
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
index 633405cc5f7..f2cca110b61 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 package com.yahoo.vespa.hosted.node.admin.noderepository;
 
+import com.google.common.collect.Sets;
 import com.yahoo.application.Networking;
 import com.yahoo.application.container.JDisc;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -8,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
 import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -17,14 +20,22 @@ import java.net.ServerSocket;
 import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 
-
+/**
+ * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository
+ * which already contains some data.
+ *
+ * @author dybdahl
+ */
 public class NodeRepositoryImplTest {
     private JDisc container;
     private int port;
+    private final Set<HostName> configServerHosts = Sets.newHashSet(new HostName("127.0.0.1"));
+
 
     private int findRandomOpenPort() throws IOException {
         try (ServerSocket socket = new ServerSocket(0)) {
@@ -42,12 +53,13 @@ public class NodeRepositoryImplTest {
     @Before
     public void startContainer() throws Exception {
         port = findRandomOpenPort();
+        System.err.println("PORT IS " + port);
         container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable);
     }
 
     private void waitForJdiscContainerToServe() throws InterruptedException {
         Instant start = Instant.now();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("foobar", "127.0.0.1", port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(Sets.newHashSet(new HostName("127.0.0.1")), port, "foobar");
         while (Instant.now().minusSeconds(120).isBefore(start)) {
             try {
                 nodeRepositoryApi.getContainersToRun();
@@ -66,11 +78,10 @@ public class NodeRepositoryImplTest {
         }
     }
 
-
     @Test
     public void testGetContainersToRunAPi() throws IOException, InterruptedException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4");
         final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun();
         assertThat(containersToRun.size(), is(1));
         final ContainerNodeSpec nodeSpec = containersToRun.get(0);
@@ -88,9 +99,9 @@ public class NodeRepositoryImplTest {
     @Test
     public void testGetContainers() throws InterruptedException, IOException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4");
         HostName hostname = new HostName("host4.yahoo.com");
-        Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainer(hostname);
+        Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname);
         assertThat(nodeSpec.isPresent(), is(true));
         assertThat(nodeSpec.get().hostname, is(hostname));
         assertThat(nodeSpec.get().containerName, is(new ContainerName("host4")));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
index f5c5601d661..4efe27743e3 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
@@ -5,10 +5,22 @@ import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
 import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy;
 import com.yahoo.vespa.orchestrator.restapi.HostApi;
+import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
-import static org.mockito.Matchers.anyString;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -16,16 +28,29 @@ import static org.mockito.Mockito.when;
 
 /**
  * @author bakksjo
+ * @author dybis
  */
 public class OrchestratorImplTest {
-    @Test
-    public void redundantResumesAreFilteredOut() throws Exception {
-        final HostApi hostApi = mock(HostApi.class);
+    private HostApi hostApi;
+    private OrchestratorImpl orchestrator;
+    private HostSuspensionApi hostSuspensionApi;
+    final String hostNameString = "host";
+    final HostName hostName = new HostName(hostNameString);
+    final List<String> hosts = new ArrayList<>();
+
+    @Before
+    public void before() {
+        hostApi = mock(HostApi.class);
         final JaxRsStrategy<HostApi> hostApiClient = new LocalPassThroughJaxRsStrategy<>(hostApi);
-        final OrchestratorImpl orchestrator = new OrchestratorImpl(hostApiClient);
-        final String hostNameString = "host";
-        final HostName hostName = new HostName(hostNameString);
 
+        hostSuspensionApi = mock(HostSuspensionApi.class);
+        final JaxRsStrategy<HostSuspensionApi> hostSuspendClient = new LocalPassThroughJaxRsStrategy<>(hostSuspensionApi);
+
+        orchestrator = new OrchestratorImpl(hostApiClient, hostSuspendClient);
+    }
+
+    @Test
+    public void testSingleOperations() throws Exception {
         // Make resume and suspend always succeed.
         when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null));
         when(hostApi.suspend(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null));
@@ -33,18 +58,43 @@ public class OrchestratorImplTest {
         orchestrator.resume(hostName);
         verify(hostApi, times(1)).resume(hostNameString);
 
-        // A subsequent resume does not cause a network trip.
-        orchestrator.resume(hostName);
-        verify(hostApi, times(1)).resume(anyString());
-
         orchestrator.suspend(hostName);
         verify(hostApi, times(1)).suspend(hostNameString);
 
-        orchestrator.resume(hostName);
-        verify(hostApi, times(2)).resume(hostNameString);
+        hosts.add(hostNameString);
+        orchestrator.resume(hosts);
+    }
 
-        // A subsequent resume does not cause a network trip.
-        orchestrator.resume(hostName);
-        verify(hostApi, times(2)).resume(anyString());
+    @Test
+    public void testListResumeOk() {
+        when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null));
+        hosts.add(hostNameString);
+        final Optional<String> resume = orchestrator.resume(hosts);
+        assertFalse(resume.isPresent());
+        verify(hostApi, times(1)).resume(hostNameString);
+    }
+
+    @Test
+    public void testListResumeFailed() {
+        when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, new HostStateChangeDenialReason("", "", "")));
+        hosts.add(hostNameString);
+        final Optional<String> resume = orchestrator.resume(hosts);
+        assertTrue(resume.isPresent());
+        assertThat(resume.get(), is("Could not resume host"));
+        verify(hostApi, times(1)).resume(hostNameString);
+    }
+
+    @Test
+    public void testListSuspendOk() throws Exception {
+        hosts.add(hostNameString);
+        when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult(null));
+        assertThat(orchestrator.suspend("parent", hosts), is(Optional.empty()));
+    }
+
+    @Test
+    public void testListSuspendFailed() throws Exception {
+        hosts.add(hostNameString);
+        when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult("no no"));
+        assertThat(orchestrator.suspend("parent", hosts), is(Optional.of("no no")));
     }
 }