diff options
5 files changed, 48 insertions, 68 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 0018dd22dd9..066118294a0 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -7,14 +7,15 @@ import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.core.SystemTimer; -import java.util.logging.Level; import java.util.Map; import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -31,14 +32,13 @@ public class StateMonitor extends AbstractComponent { public enum Status {up, down, initializing} private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); - private final Thread thread; + private final ScheduledExecutorService executor; private final Timer timer; private final long snapshotIntervalMs; private volatile long lastSnapshotTimeMs; private volatile MetricSnapshot snapshot; private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); - private final AtomicBoolean stopped = new AtomicBoolean(false); /** For testing */ public StateMonitor() { @@ -59,17 +59,22 @@ public class StateMonitor extends AbstractComponent { StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)), Status.valueOf(config.initialStatus()), - timer, threadFactory); + timer, threadFactory, true); } /* Public for testing only */ - public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) { + public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory, + boolean start) { this.timer = timer; this.snapshotIntervalMs = snapshotIntervalMS; this.lastSnapshotTimeMs = timer.currentTimeMillis(); this.status = status; - thread = threadFactory.newThread(this::run); - thread.start(); + this.executor = Executors.newSingleThreadScheduledExecutor(threadFactory); + + if (start) { + executor.scheduleAtFixedRate(this::updateSnapshot, snapshotIntervalMS, + snapshotIntervalMS, TimeUnit.MILLISECONDS); + } } /** Returns a metric consumer for jDisc which will write metrics back to this */ @@ -96,28 +101,11 @@ public class StateMonitor extends AbstractComponent { /** Returns the interval between each metrics snapshot used by this */ public long getSnapshotIntervalMillis() { return snapshotIntervalMs; } - /** NOTE: For unit testing only. May lead to undefined behaviour if StateMonitor thread is running simultaneously **/ - boolean checkTime() { + /** NOTE: Non-private for unit testing only. **/ + void updateSnapshot() { long now = timer.currentTimeMillis(); - if (now < lastSnapshotTimeMs + snapshotIntervalMs) { - return false; - } snapshot = createSnapshot(lastSnapshotTimeMs, now); lastSnapshotTimeMs = now; - return true; - } - - private void run() { - log.finest("StateMonitor started."); - try { - synchronized (stopped) { - while (!stopped.get()) { - checkTime(); - stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); - } - } - } catch (InterruptedException e) { } - log.finest("StateMonitor stopped."); } private MetricSnapshot createSnapshot(long fromMillis, long toMillis) { @@ -148,14 +136,12 @@ public class StateMonitor extends AbstractComponent { @Override public void deconstruct() { - synchronized (stopped) { - stopped.set(true); - stopped.notifyAll(); - } + executor.shutdown(); try { - thread.join(5000); + executor.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException e) { } - if (thread.isAlive()) { + + if (!executor.isTerminated()) { log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index e7a9a1442f3..d26d7f7f013 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -1,14 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; -import static org.junit.Assert.*; - import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author bratseth */ @@ -142,7 +144,7 @@ public class VipStatusTestCase { Thread thread = new Thread(runnable, "StateMonitor"); thread.setDaemon(true); return thread; - }); + }, true); } private static void removeFromRotation(String[] clusters, VipStatus v) { diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index 98dfb2e281b..ef700597537 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -17,7 +17,6 @@ import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.PACKET_SEPAR import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.STATUS_CODE_KEY; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.STATUS_MSG_KEY; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.TIMESTAMP_KEY; -import static com.yahoo.container.jdisc.state.StateHandlerTestBase.SNAPSHOT_INTERVAL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -123,7 +122,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { } private List<JsonNode> incrementTimeAndGetJsonPackets() throws Exception { - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); String response = requestAsString("http://localhost/metrics-packets"); return toJsonPackets(response); diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java index d53b189f932..26a2f817acc 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java @@ -34,13 +34,13 @@ public class StateHandlerTest extends StateHandlerTestBase { public void testReportIncludesMetricsAfterSnapshot() throws Exception { metric.add("foo", 1, null); metric.set("bar", 4, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json1 = requestAsJson("http://localhost/state/v1/metrics"); assertEquals(json1.toString(), "up", json1.get("status").get("code").asText()); assertEquals(json1.toString(), 2, json1.get("metrics").get("values").size()); metric.add("fuz", 1, metric.createContext(new HashMap<>(0))); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json2 = requestAsJson("http://localhost/state/v1/metrics"); assertEquals(json2.toString(), "up", json2.get("status").get("code").asText()); assertEquals(json2.toString(), 3, json2.get("metrics").get("values").size()); @@ -61,7 +61,7 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.add(metricName, 2, metricContext); // Change it to a gauge metric metric.set(metricName, 9, metricContext); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); MetricValue resultingMetric = monitor.snapshot().iterator().next().getValue().get(metricName); assertEquals(GaugeMetric.class, resultingMetric.getClass()); assertEquals("Value was reset and produces the last gauge value", @@ -74,7 +74,7 @@ public class StateHandlerTest extends StateHandlerTestBase { // Change it to a count metric metric.add(metricName, 1, metricContext); metric.add(metricName, 2, metricContext); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); MetricValue resultingMetric = monitor.snapshot().iterator().next().getValue().get(metricName); assertEquals(CountMetric.class, resultingMetric.getClass()); assertEquals("Value was reset, and changed to add semantics giving 1+2", @@ -88,7 +88,7 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.set("bar", 5, null); metric.set("bar", 7, null); metric.set("bar", 2, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); @@ -102,7 +102,7 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.add("foo", 1, null); metric.add("foo", 2, null); metric.add("foo", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); @@ -113,7 +113,7 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void testReadabilityOfJsonReport() throws Exception { metric.add("foo", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); assertEquals("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + @@ -138,7 +138,7 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.set("bar", 3, ctx); metric.set("bar", 4, ctx); metric.set("bar", 5, ctx); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); assertEquals("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + @@ -178,10 +178,10 @@ public class StateHandlerTest extends StateHandlerTestBase { public void testNotAggregatingCountsBeyondSnapshots() throws Exception { metric.add("foo", 1, null); metric.add("foo", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); metric.add("foo", 2, null); metric.add("foo", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); @@ -194,21 +194,19 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.add("foo", 1, null); metric.set("bar", 3, null); // At this time we should not have done any snapshotting - incrementCurrentTimeAndAssertNoSnapshot(SNAPSHOT_INTERVAL - 1); { JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertFalse(json.toString(), json.get("metrics").has("snapshot")); } // At this time first snapshot should have been generated - incrementCurrentTimeAndAssertSnapshot(1); + advanceToNextSnapshot(); { JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertTrue(json.toString(), json.get("metrics").has("snapshot")); assertEquals(0.0, json.get("metrics").get("snapshot").get("from").asDouble(), 0.00001); assertEquals(300.0, json.get("metrics").get("snapshot").get("to").asDouble(), 0.00001); } - // No new snapshot at this time - incrementCurrentTimeAndAssertNoSnapshot(SNAPSHOT_INTERVAL - 1); + // No new snapshot { JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertTrue(json.toString(), json.get("metrics").has("snapshot")); @@ -216,7 +214,7 @@ public class StateHandlerTest extends StateHandlerTestBase { assertEquals(300.0, json.get("metrics").get("snapshot").get("to").asDouble(), 0.00001); } // A new snapshot - incrementCurrentTimeAndAssertSnapshot(1); + advanceToNextSnapshot(); { JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertTrue(json.toString(), json.get("metrics").has("snapshot")); @@ -229,10 +227,10 @@ public class StateHandlerTest extends StateHandlerTestBase { public void testFreshStartOfValuesBeyondSnapshot() throws Exception { metric.set("bar", 4, null); metric.set("bar", 5, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); metric.set("bar", 4, null); metric.set("bar", 2, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); @@ -243,8 +241,8 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void snapshotsPreserveLastGaugeValue() throws Exception { metric.set("bar", 4, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); JsonNode metricValues = getFirstMetricValueNode(json); assertEquals(json.toString(), 4, metricValues.get("last").asDouble(), 0.001); @@ -266,10 +264,10 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void gaugeSnapshotsTracksCountMinMaxAvgPerPeriod() throws Exception { metric.set("bar", 10000, null); // Ensure any cross-snapshot noise is visible - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); metric.set("bar", 20, null); metric.set("bar", 40, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/all"); JsonNode metricValues = getFirstMetricValueNode(json); assertEquals(json.toString(), 40, metricValues.get("last").asDouble(), 0.001); @@ -294,7 +292,7 @@ public class StateHandlerTest extends StateHandlerTestBase { metric.set("serverTotalSuccessfulResponseLatency", 20, context1); metric.set("serverTotalSuccessfulResponseLatency", 40, context2); metric.add("random", 3, context1); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); + advanceToNextSnapshot(); JsonNode json = requestAsJson("http://localhost/state/v1/health"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 2, json.get("metrics").get("values").size()); @@ -324,9 +322,4 @@ public class StateHandlerTest extends StateHandlerTestBase { JsonNode version = root.get("version"); assertEquals(Vtag.currentVersion.toString(), version.asText()); } - - private void incrementCurrentTimeAndAssertNoSnapshot(long val) { - currentTimeMillis.addAndGet(val); - assertFalse("Expected no snapshot", monitor.checkTime());; - } } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index 8a1640e2c0e..6b44115016f 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -110,9 +110,9 @@ public class StateHandlerTestBase { return mapper.readTree(mapper.getFactory().createParser(requestAsString(requestUri))); } - void incrementCurrentTimeAndAssertSnapshot(long val) { - currentTimeMillis.addAndGet(val); - assertTrue("Expected a new snapshot to be generated", monitor.checkTime()); + void advanceToNextSnapshot() { + currentTimeMillis.addAndGet(SNAPSHOT_INTERVAL); + monitor.updateSnapshot(); } } |