Add service model snapshot latency metric

6 files changed, 138 insertions, 13 deletions

diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml
index 13602c60cf8..80958ca9bee 100644
--- a/service-monitor/pom.xml
+++ b/service-monitor/pom.xml
@@ -37,6 +37,12 @@
         </dependency>
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
+            <artifactId>jdisc_core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
             <artifactId>configdefinitions</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
index f1d8e31d26c..0eaccf50e29 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
@@ -5,8 +5,11 @@ import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.model.api.SuperModelProvider;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Timer;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.service.monitor.internal.ServiceMonitorMetrics;
 
 import java.util.Collections;
 import java.util.List;
@@ -20,14 +23,17 @@ public class ServiceMonitorImpl implements ServiceMonitor {
     private final Zone zone;
     private final List<String> configServerHosts;
     private final SlobrokMonitorManager slobrokMonitorManager = new SlobrokMonitorManager();
-    private final SuperModelListenerImpl superModelListener =
-            new SuperModelListenerImpl(slobrokMonitorManager);
+    private final SuperModelListenerImpl superModelListener;
 
     @Inject
     public ServiceMonitorImpl(SuperModelProvider superModelProvider,
-                              ConfigserverConfig configserverConfig) {
+                              ConfigserverConfig configserverConfig,
+                              Metric metric,
+                              Timer timer) {
         this.zone = superModelProvider.getZone();
         this.configServerHosts = toConfigServerList(configserverConfig);
+        ServiceMonitorMetrics metrics = new ServiceMonitorMetrics(metric, timer);
+        this.superModelListener = new SuperModelListenerImpl(slobrokMonitorManager, metrics);
         superModelListener.start(superModelProvider);
     }
 
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
index 18f1e3ede6d..a1dd6d5a265 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
@@ -7,6 +7,8 @@ import com.yahoo.config.model.api.SuperModelListener;
 import com.yahoo.config.model.api.SuperModelProvider;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.service.monitor.internal.LatencyMeasurement;
+import com.yahoo.vespa.service.monitor.internal.ServiceMonitorMetrics;
 
 import java.util.List;
 import java.util.logging.Logger;
@@ -14,14 +16,18 @@ import java.util.logging.Logger;
 public class SuperModelListenerImpl implements SuperModelListener {
     private static final Logger logger = Logger.getLogger(SuperModelListenerImpl.class.getName());
 
+    private final ServiceMonitorMetrics metrics;
+
     // superModel and slobrokMonitorManager are always updated together
     // and atomically using this monitor.
     private final Object monitor = new Object();
     private final SlobrokMonitorManager slobrokMonitorManager;
     private SuperModel superModel;
 
-    SuperModelListenerImpl(SlobrokMonitorManager slobrokMonitorManager) {
+    SuperModelListenerImpl(SlobrokMonitorManager slobrokMonitorManager,
+                           ServiceMonitorMetrics metrics) {
         this.slobrokMonitorManager = slobrokMonitorManager;
+        this.metrics = metrics;
     }
 
     void start(SuperModelProvider superModelProvider) {
@@ -55,14 +61,18 @@ public class SuperModelListenerImpl implements SuperModelListener {
     ServiceModel createServiceModelSnapshot(Zone zone, List<String> configServerHostnames) {
         ModelGenerator modelGenerator = new ModelGenerator();
 
-        // TODO: Add latency and calls-per-second metrics
-        // If calculating the service model is too expensive per call, then
-        // cache the generated snapshot, invalidate after X seconds.
-        // WARNING: The slobrok monitor manager may be out-of-sync with super model (no locking)
-        return modelGenerator.toServiceModel(
-                superModel,
-                zone,
-                configServerHostnames,
-                slobrokMonitorManager);
+        try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) {
+            // Reference 'measurement' in a dummy statement, otherwise the compiler
+            // complains about "auto-closeable resource is never referenced in body of
+            // corresponding try statement". Why hasn't javac fixed this!?
+            measurement.hashCode();
+
+            // WARNING: The slobrok monitor manager may be out-of-sync with super model (no locking)
+            return modelGenerator.toServiceModel(
+                    superModel,
+                    zone,
+                    configServerHostnames,
+                    slobrokMonitorManager);
+        }
     }
 }
\ No newline at end of file
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/LatencyMeasurement.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/LatencyMeasurement.java
new file mode 100644
index 00000000000..b4a8621448b
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/LatencyMeasurement.java
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.monitor.internal;
+
+import com.yahoo.jdisc.Timer;
+
+import java.util.function.Consumer;
+
+public class LatencyMeasurement implements AutoCloseable {
+    private final Timer timer;
+    private Consumer<Double> elapsedSecondsConsumer;
+    private long startMillis;
+
+    LatencyMeasurement(Timer timer, Consumer<Double> elapsedSecondsConsumer) {
+        this.timer = timer;
+        this.elapsedSecondsConsumer = elapsedSecondsConsumer;
+    }
+
+    LatencyMeasurement start() {
+        startMillis = timer.currentTimeMillis();
+        return this;
+    }
+
+    @Override
+    public void close() {
+        if (elapsedSecondsConsumer != null) {
+            double elapsedSeconds = forceNonNegative(timer.currentTimeMillis() - startMillis);
+            elapsedSecondsConsumer.accept(elapsedSeconds);
+            elapsedSecondsConsumer = null;
+        }
+    }
+
+    private static double forceNonNegative(double d) {
+        return d > 0 ? d : 0;
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetrics.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetrics.java
new file mode 100644
index 00000000000..e55a66473bb
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetrics.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.monitor.internal;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Timer;
+
+import java.util.function.Consumer;
+
+public class ServiceMonitorMetrics {
+    public static String SERVICE_MODEL_METRIC_PREFIX = "serviceModel.";
+
+    private final Metric metric;
+    private final Timer timer;
+
+    public ServiceMonitorMetrics(Metric metric, Timer timer) {
+        this.metric = metric;
+        this.timer = timer;
+    }
+
+    public LatencyMeasurement startServiceModelSnapshotLatencyMeasurement() {
+        Consumer<Double> atCompletion = elapsedSeconds ->
+                setValue(metricKey("snapshot.latency"), elapsedSeconds);
+        return new LatencyMeasurement(timer, atCompletion).start();
+    }
+
+    private static String metricKey(String suffix) {
+        return SERVICE_MODEL_METRIC_PREFIX + suffix;
+    }
+
+    private void setValue(String key, Number number) {
+        setValue(key, number, null);
+    }
+
+    private void setValue(String key, Number number, Metric.Context context) {
+        metric.set(key, number, context);
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetricsTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetricsTest.java
new file mode 100644
index 00000000000..c5f994fc5b1
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorMetricsTest.java
@@ -0,0 +1,29 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.monitor.internal;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Timer;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ServiceMonitorMetricsTest {
+    @Test
+    public void testTryWithResources() {
+        Metric metric = mock(Metric.class);
+        Timer timer = mock(Timer.class);
+        ServiceMonitorMetrics metrics = new ServiceMonitorMetrics(metric, timer);
+
+        when(timer.currentTimeMillis()).thenReturn(Long.valueOf(0), Long.valueOf(10));
+
+        try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) {
+            measurement.hashCode();
+        }
+
+        verify(metric).set("serviceModel.snapshot.latency", 10.0, null);
+    }
+
+}
\ No newline at end of file