diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2017-08-16 11:29:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-16 11:29:39 +0200 |
commit | 1cc1400ed990fdd5c1fbd5bad4816e77ee00fb24 (patch) | |
tree | ea6bb03c987a408db0b6a70b72a94d0efd6817c2 /node-admin | |
parent | 4b97abc50d84c9d9c6138a5a655ebe2704436f7b (diff) | |
parent | 85f23e91fa4457e352a74d40d0d3f144a1bfc1ef (diff) |
Merge pull request #3083 from vespa-engine/freva/check-node-spec-once-an-hour
Verify node spec once an hour in node-admin
Diffstat (limited to 'node-admin')
6 files changed, 44 insertions, 4 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 6cfa5efcc81..cffbaf84159 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -296,6 +297,17 @@ public class StorageMaintainer { getMaintenanceThrottlerFor(containerName).reset(); } + /** + * Runs node-maintainer's SpecVerifier and returns its output + * @throws RuntimeException if exit code != 0 + */ + public String getHardwardDivergence() { + String configServers = environment.getConfigServerHosts().stream() + .map(configServer -> "http://" + configServer + ":" + 4080) + .collect(Collectors.joining(",")); + return executeMaintainer("com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier", configServers); + } + private String getKernelVersion() throws IOException, InterruptedException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 093551b6348..240de94664a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; @@ -17,7 +20,12 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -40,6 +48,8 @@ public class NodeAdminStateUpdater extends AbstractComponent { private final Object monitor = new Object(); private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); + private final ScheduledExecutorService specVerifierScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("specverifier")); private Thread loopThread; private final NodeRepository nodeRepository; @@ -54,6 +64,7 @@ public class NodeAdminStateUpdater extends AbstractComponent { public NodeAdminStateUpdater( final NodeRepository nodeRepository, final NodeAdmin nodeAdmin, + Optional<StorageMaintainer> storageMaintainer, Clock clock, Orchestrator orchestrator, String dockerHostHostName) { @@ -64,6 +75,9 @@ public class NodeAdminStateUpdater extends AbstractComponent { this.orchestrator = orchestrator; this.dockerHostHostName = dockerHostHostName; this.lastTick = clock.instant(); + + storageMaintainer.ifPresent(maintainer -> specVerifierScheduler.scheduleWithFixedDelay(() -> + updateHardwareDivergence(maintainer), 5, 60, TimeUnit.MINUTES)); } private String objectToString() { @@ -83,6 +97,18 @@ public class NodeAdminStateUpdater extends AbstractComponent { return debug; } + private void updateHardwareDivergence(StorageMaintainer maintainer) { + if (currentState != RESUMED) return; + + try { + String hardwareDivergence = maintainer.getHardwardDivergence(); + NodeAttributes nodeAttributes = new NodeAttributes().withHardwareDivergence(hardwareDivergence); + nodeRepository.updateNodeAttributes(dockerHostHostName, nodeAttributes); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to report hardware divergence", e); + } + } + public boolean setResumeStateAndCheckIfResumed(State wantedState) { synchronized (monitor) { if (this.wantedState != wantedState) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java index 56c15804179..d61dd3c591b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -75,7 +75,7 @@ public class ComponentsProviderImpl implements ComponentsProvider { storageMaintainer, environment, clock, aclMaintainer); NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver, aclMaintainer, clock); - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, clock, orchestrator, baseHostName); + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, storageMaintainer, clock, orchestrator, baseHostName); nodeAdminStateUpdater.start(NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS); metricReceiverWrapper = metricReceiver; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java index 12d304712f8..166a2d3f70e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java @@ -36,7 +36,7 @@ public class ComponentsProviderWithMocks implements ComponentsProvider { (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperationsMock, Optional.empty(), environment, Clock.systemUTC(), Optional.empty()); private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, Optional.empty(), 100, mr, Optional.empty(), Clock.systemUTC()); - private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Clock.systemUTC(), orchestratorMock, "localhost.test.yahoo.com"); + private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.empty(), Clock.systemUTC(), orchestratorMock, "localhost.test.yahoo.com"); public ComponentsProviderWithMocks() { nodeAdminStateUpdater.start(10); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 735e8a574f7..714e8df64ec 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -60,7 +60,8 @@ public class DockerTester implements AutoCloseable { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperations, Optional.of(storageMaintainer), environment, clock, Optional.empty()); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.of(storageMaintainer), 100, mr, Optional.empty(), Clock.systemUTC()); - updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, clock, orchestratorMock, "basehostname"); + updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.of(storageMaintainer), + clock, orchestratorMock, "basehostname"); updater.start(5); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index f1f269adf5f..28ed462d73b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -14,6 +14,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -41,7 +42,7 @@ public class NodeAdminStateUpdaterTest { private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); private final Orchestrator orchestrator = mock(Orchestrator.class); private final NodeAdminStateUpdater refresher = spy(new NodeAdminStateUpdater( - nodeRepository, nodeAdmin, clock, orchestrator, parentHostname)); + nodeRepository, nodeAdmin, Optional.empty(), clock, orchestrator, parentHostname)); @Test |