Restart container if different wanted and current ContainerResources

2 files changed, 48 insertions, 0 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 453012e9791..7b61436837c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerException;
 import com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException;
@@ -318,6 +319,13 @@ public class NodeAgentImpl implements NodeAgent {
         if (!existingContainer.state.isRunning()) {
             return Optional.of("Container no longer running");
         }
+
+        ContainerResources wantedContainerResources = ContainerResources.from(
+                nodeSpec.minCpuCores, nodeSpec.minMainMemoryAvailableGb);
+        if (!wantedContainerResources.equals(existingContainer.flavor)) {
+            return Optional.of("Container should be running with different resource allocation, wanted: " +
+                    wantedContainerResources + ", actual: " + existingContainer.flavor);
+        }
         return Optional.empty();
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index ceaa1d58f92..cce000f858a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -6,6 +6,7 @@ import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
 import com.yahoo.vespa.hosted.dockerapi.ContainerStatsImpl;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
@@ -229,6 +230,44 @@ public class NodeAgentImplTest {
     }
 
     @Test
+    public void containerIsRestartedIfFlavorChanged() throws Exception {
+        final long wantedRestartGeneration = 1;
+        final long currentRestartGeneration = 1;
+        ContainerNodeSpec.Builder specBuilder = nodeSpecBuilder
+                .wantedDockerImage(dockerImage)
+                .currentDockerImage(dockerImage)
+                .nodeState(Node.State.active)
+                .wantedVespaVersion(vespaVersion)
+                .vespaVersion(vespaVersion)
+                .wantedRestartGeneration(wantedRestartGeneration)
+                .currentRestartGeneration(currentRestartGeneration);
+
+        NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+        ContainerNodeSpec firstSpec = specBuilder.build();
+        ContainerNodeSpec secondSpec = specBuilder.minDiskAvailableGb(200).build();
+        ContainerNodeSpec thirdSpec = specBuilder.minCpuCores(4).build();
+
+        when(nodeRepository.getContainerNodeSpec(hostName))
+                .thenReturn(Optional.of(firstSpec))
+                .thenReturn(Optional.of(secondSpec))
+                .thenReturn(Optional.of(thirdSpec));
+        when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true);
+        when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L));
+
+        nodeAgent.converge();
+        nodeAgent.converge();
+        nodeAgent.converge();
+
+        InOrder inOrder = inOrder(orchestrator, dockerOperations);
+        inOrder.verify(orchestrator).resume(any(String.class));
+        inOrder.verify(orchestrator).resume(any(String.class));
+        inOrder.verify(orchestrator).suspend(any(String.class));
+        inOrder.verify(dockerOperations).removeContainer(any());
+        inOrder.verify(dockerOperations).startContainer(eq(containerName), eq(thirdSpec));
+        inOrder.verify(orchestrator).resume(any(String.class));
+    }
+
+    @Test
     public void noRestartIfOrchestratorSuspendFails() throws Exception {
         final long wantedRestartGeneration = 2;
         final long currentRestartGeneration = 1;
@@ -611,6 +650,7 @@ public class NodeAgentImplTest {
                 Optional.of(new Container(
                         hostName,
                         dockerImage,
+                        ContainerResources.from(MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB),
                         containerName,
                         isRunning ? Container.State.RUNNING : Container.State.EXITED,
                         isRunning ? 1 : 0)) :