Merge pull request #3083 from vespa-engine/freva/check-node-spec-once-an-hour

Verify node spec once an hour in node-admin

6 files changed, 44 insertions, 4 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 6cfa5efcc81..cffbaf84159 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
@@ -296,6 +297,17 @@ public class StorageMaintainer {
         getMaintenanceThrottlerFor(containerName).reset();
     }
 
+    /**
+     * Runs node-maintainer's SpecVerifier and returns its output
+     * @throws RuntimeException if exit code != 0
+     */
+    public String getHardwardDivergence() {
+        String configServers = environment.getConfigServerHosts().stream()
+                .map(configServer -> "http://" +  configServer + ":" + 4080)
+                .collect(Collectors.joining(","));
+        return executeMaintainer("com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier", configServers);
+    }
+
 
 
     private String getKernelVersion() throws IOException, InterruptedException {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
index 093551b6348..240de94664a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
 import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException;
@@ -17,7 +20,12 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -40,6 +48,8 @@ public class NodeAdminStateUpdater extends AbstractComponent {
     private final Object monitor = new Object();
 
     private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName());
+    private final ScheduledExecutorService specVerifierScheduler =
+            Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("specverifier"));
     private Thread loopThread;
 
     private final NodeRepository nodeRepository;
@@ -54,6 +64,7 @@ public class NodeAdminStateUpdater extends AbstractComponent {
     public NodeAdminStateUpdater(
             final NodeRepository nodeRepository,
             final NodeAdmin nodeAdmin,
+            Optional<StorageMaintainer> storageMaintainer,
             Clock clock,
             Orchestrator orchestrator,
             String dockerHostHostName) {
@@ -64,6 +75,9 @@ public class NodeAdminStateUpdater extends AbstractComponent {
         this.orchestrator = orchestrator;
         this.dockerHostHostName = dockerHostHostName;
         this.lastTick = clock.instant();
+
+        storageMaintainer.ifPresent(maintainer -> specVerifierScheduler.scheduleWithFixedDelay(() ->
+                updateHardwareDivergence(maintainer), 5, 60, TimeUnit.MINUTES));
     }
 
     private String objectToString() {
@@ -83,6 +97,18 @@ public class NodeAdminStateUpdater extends AbstractComponent {
         return debug;
     }
 
+    private void updateHardwareDivergence(StorageMaintainer maintainer) {
+        if (currentState != RESUMED) return;
+
+        try {
+            String hardwareDivergence = maintainer.getHardwardDivergence();
+            NodeAttributes nodeAttributes = new NodeAttributes().withHardwareDivergence(hardwareDivergence);
+            nodeRepository.updateNodeAttributes(dockerHostHostName, nodeAttributes);
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING, "Failed to report hardware divergence", e);
+        }
+    }
+
     public boolean setResumeStateAndCheckIfResumed(State wantedState) {
         synchronized (monitor) {
             if (this.wantedState != wantedState) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
index 56c15804179..d61dd3c591b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java
@@ -75,7 +75,7 @@ public class ComponentsProviderImpl implements ComponentsProvider {
                         storageMaintainer, environment, clock, aclMaintainer);
         NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer,
                 NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver, aclMaintainer, clock);
-        nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, clock, orchestrator, baseHostName);
+        nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, storageMaintainer, clock, orchestrator, baseHostName);
         nodeAdminStateUpdater.start(NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS);
 
         metricReceiverWrapper = metricReceiver;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java
index 12d304712f8..166a2d3f70e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java
@@ -36,7 +36,7 @@ public class ComponentsProviderWithMocks implements ComponentsProvider {
             (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock,
                     dockerOperationsMock, Optional.empty(), environment, Clock.systemUTC(), Optional.empty());
     private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, Optional.empty(), 100, mr, Optional.empty(), Clock.systemUTC());
-    private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Clock.systemUTC(), orchestratorMock, "localhost.test.yahoo.com");
+    private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.empty(), Clock.systemUTC(), orchestratorMock, "localhost.test.yahoo.com");
 
     public ComponentsProviderWithMocks() {
         nodeAdminStateUpdater.start(10);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 735e8a574f7..714e8df64ec 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -60,7 +60,8 @@ public class DockerTester implements AutoCloseable {
         Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock,
                 orchestratorMock, dockerOperations, Optional.of(storageMaintainer), environment, clock, Optional.empty());
         nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.of(storageMaintainer), 100, mr, Optional.empty(), Clock.systemUTC());
-        updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, clock, orchestratorMock, "basehostname");
+        updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.of(storageMaintainer),
+                clock, orchestratorMock, "basehostname");
         updater.start(5);
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
index f1f269adf5f..28ed462d73b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
@@ -14,6 +14,7 @@ import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -41,7 +42,7 @@ public class NodeAdminStateUpdaterTest {
     private final NodeAdmin nodeAdmin = mock(NodeAdmin.class);
     private final Orchestrator orchestrator = mock(Orchestrator.class);
     private final NodeAdminStateUpdater refresher = spy(new NodeAdminStateUpdater(
-            nodeRepository, nodeAdmin, clock, orchestrator, parentHostname));
+            nodeRepository, nodeAdmin, Optional.empty(), clock, orchestrator, parentHostname));
 
 
     @Test