From 0e559539cbd6e604722e47574c20bd0c51ea9e86 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 1 Apr 2021 15:08:22 +0200 Subject: Add remote task queue size metric in cluster controller --- .../clustercontroller/core/FleetController.java | 34 ++++++++++++---------- .../clustercontroller/core/MetricUpdater.java | 6 +++- .../core/ClusterFeedBlockTest.java | 2 +- .../core/FleetControllerTest.java | 2 +- .../clustercontroller/core/MetricReporterTest.java | 8 +++-- .../clustercontroller/core/StateChangeTest.java | 2 +- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 2f79393a020..75eea998346 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -170,7 +170,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd StatusPageServerInterface statusPageServer, MetricReporter metricReporter) throws Exception { Timer timer = new RealTimer(); - MetricUpdater metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex, options.clusterName); EventLog log = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster( options.clusterName, @@ -757,21 +757,25 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } private boolean processNextQueuedRemoteTask() { - if ( ! remoteTasks.isEmpty()) { - final RemoteClusterControllerTask.Context context = createRemoteTaskProcessingContext(); - final RemoteClusterControllerTask task = remoteTasks.poll(); - log.finest(() -> String.format("Processing remote task of type '%s'", task.getClass().getName())); - task.doRemoteFleetControllerTask(context); - if (taskMayBeCompletedImmediately(task)) { - log.finest(() -> String.format("Done processing remote task of type '%s'", task.getClass().getName())); - task.notifyCompleted(); - } else { - log.finest(() -> String.format("Remote task of type '%s' queued until state recomputation", task.getClass().getName())); - tasksPendingStateRecompute.add(task); - } - return true; + metricUpdater.updateRemoteTaskQueueSize(remoteTasks.size()); + + RemoteClusterControllerTask task = remoteTasks.poll(); + if (task == null) { + return false; } - return false; + + final RemoteClusterControllerTask.Context context = createRemoteTaskProcessingContext(); + log.finest(() -> String.format("Processing remote task of type '%s'", task.getClass().getName())); + task.doRemoteFleetControllerTask(context); + if (taskMayBeCompletedImmediately(task)) { + log.finest(() -> String.format("Done processing remote task of type '%s'", task.getClass().getName())); + task.notifyCompleted(); + } else { + log.finest(() -> String.format("Remote task of type '%s' queued until state recomputation", task.getClass().getName())); + tasksPendingStateRecompute.add(task); + } + + return true; } private boolean taskMayBeCompletedImmediately(RemoteClusterControllerTask task) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index 6a80772904f..94b6f412ce6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -14,9 +14,10 @@ public class MetricUpdater { private final ComponentMetricReporter metricReporter; - public MetricUpdater(MetricReporter metricReporter, int controllerIndex) { + public MetricUpdater(MetricReporter metricReporter, int controllerIndex, String clusterName) { this.metricReporter = new ComponentMetricReporter(metricReporter, "cluster-controller."); this.metricReporter.addDimension("controller-index", String.valueOf(controllerIndex)); + this.metricReporter.addDimension("clusterid", clusterName); } public MetricReporter.Context createContext(Map dimensions) { @@ -99,4 +100,7 @@ public class MetricUpdater { metricReporter.add("node-event", 1); } + public void updateRemoteTaskQueueSize(int size) { + metricReporter.set("remote-task-queue.size", size); + } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 56d567db035..c4db6b9525b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -49,7 +49,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { } communicator = new DummyCommunicator(nodes, timer); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); EventLog eventLog = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index b238a9eadcf..1d7b6886222 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -153,7 +153,7 @@ public abstract class FleetControllerTest implements Waiter { FleetController createFleetController(boolean useFakeTimer, FleetControllerOptions options, boolean startThread, StatusPageServerInterface status) throws Exception { Objects.requireNonNull(status, "status server cannot be null"); Timer timer = useFakeTimer ? this.timer : new RealTimer(); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); EventLog log = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster( options.clusterName, diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java index 3e44995dc78..aef223d7aee 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java @@ -21,9 +21,11 @@ import static org.mockito.hamcrest.MockitoHamcrest.doubleThat; public class MetricReporterTest { + private static final String CLUSTER_NAME = "foo"; + private static class Fixture { final MetricReporter mockReporter = mock(MetricReporter.class); - final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0); + final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0, CLUSTER_NAME); final ClusterFixture clusterFixture; Fixture() { @@ -42,8 +44,8 @@ public class MetricReporterTest { private static HasMetricContext.Dimension[] withClusterDimension() { // Dimensions that are always present HasMetricContext.Dimension controllerDim = withDimension("controller-index", "0"); - HasMetricContext.Dimension clusterDim = withDimension("cluster", "foo"); - HasMetricContext.Dimension clusteridDim = withDimension("clusterid", "foo"); + HasMetricContext.Dimension clusterDim = withDimension("cluster", CLUSTER_NAME); + HasMetricContext.Dimension clusteridDim = withDimension("clusterid", CLUSTER_NAME); return new HasMetricContext.Dimension[] { controllerDim, clusterDim, clusteridDim }; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index af39e43bb36..2b71f909115 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -50,7 +50,7 @@ public class StateChangeTest extends FleetControllerTest { } communicator = new DummyCommunicator(nodes, timer); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); eventLog = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog); -- cgit v1.2.3