Make richer latency stats

Makes a LatencyStats which provides some useful metrics, best explained there
and in LatencyMetrics.  This includes latency metrics, the "QPS" (e.g. the
number of acquire() per second), and load metrics.

Unfortunately I had to move from atomics to synchronized to accomplish this,
but I see no other way.

4 files changed, 14 insertions, 20 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index b73c6ed7b17..1cf1bb91d0c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -245,22 +245,16 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
                     metric.set("lockAttempt.release", lockMetrics.getAndResetReleaseCount(), context);
                     metric.set("lockAttempt.releaseFailed", lockMetrics.getAndResetReleaseFailedCount(), context);
 
-                    metric.set("lockAttempt.acquireNow", lockMetrics.getAcquiringNow(), context);
-                    metric.set("lockAttempt.lockedNow", lockMetrics.getLockedNow(), context);
-
                     setLockLatencyMetrics("acquire", lockMetrics.getAndResetAcquireLatencyMetrics(), context);
                     setLockLatencyMetrics("locked", lockMetrics.getAndResetLockedLatencyMetrics(), context);
                 });
     }
 
     private void setLockLatencyMetrics(String name, LatencyMetrics latencyMetrics, Metric.Context context) {
-        metric.set("lockAttempt." + name + "Latency", roundDouble(latencyMetrics.averageInSeconds(), 3), context);
-        metric.set("lockAttempt." + name + "Load", roundDouble(latencyMetrics.load(), 3), context);
-    }
-
-    private double roundDouble(double value, int decimalPlaces) {
-        double factor = Math.pow(10, decimalPlaces);
-        return Math.round(value * factor) / factor;
+        metric.set("lockAttempt." + name + "Latency", latencyMetrics.latencySeconds(), context);
+        metric.set("lockAttempt." + name + "MaxActiveLatency", latencyMetrics.maxActiveLatencySeconds(), context);
+        metric.set("lockAttempt." + name + "Hz", latencyMetrics.startHz(), context);
+        metric.set("lockAttempt." + name + "Load", latencyMetrics.load(), context);
     }
 
     private void updateDockerMetrics(NodeList nodes) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java
index 3d605201aaa..4fb760af2b2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java
@@ -61,9 +61,6 @@ public class LocksResponse extends HttpResponse {
             lockPathCursor.setLong("releaseCount", lockMetrics.getCumulativeReleaseCount());
             lockPathCursor.setLong("releaseFailedCount", lockMetrics.getCumulativeReleaseFailedCount());
 
-            lockPathCursor.setLong("acquireNow", lockMetrics.getAcquiringNow());
-            lockPathCursor.setLong("lockedNow", lockMetrics.getAcquiringNow());
-
             setLatency(lockPathCursor, "acquire", lockMetrics.getAcquireLatencyMetrics());
             setLatency(lockPathCursor, "locked", lockMetrics.getLockedLatencyMetrics());
         });
@@ -102,9 +99,10 @@ public class LocksResponse extends HttpResponse {
     }
 
     private static void setLatency(Cursor cursor, String name, LatencyMetrics latencyMetrics) {
-        cursor.setDouble(name + "LatencyAverage", roundDouble(latencyMetrics.averageInSeconds(), 3));
-        cursor.setDouble(name + "LatencyCount", latencyMetrics.count());
-        cursor.setDouble(name + "LatencyLoad", roundDouble(latencyMetrics.load(), 3));
+        cursor.setDouble(name + "Latency", latencyMetrics.latencySeconds());
+        cursor.setDouble(name + "MaxActiveLatency", latencyMetrics.maxActiveLatencySeconds());
+        cursor.setDouble(name + "Hz", latencyMetrics.endHz());
+        cursor.setDouble(name + "Load", latencyMetrics.load());
     }
 
     private static double roundDouble(double value, int decimalPlaces) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 9ff691236d7..715fc0b9915 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -159,11 +159,13 @@ public class MetricsReporterTest {
         verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.locked", 3);
         verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.release", 3);
         verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.releaseFailed", 0);
-        verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.acquireNow", 0);
-        verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.lockedNow", 0);
         metric.remove("lockAttempt.acquireLatency");
+        metric.remove("lockAttempt.acquireMaxActiveLatency");
+        metric.remove("lockAttempt.acquireHz");
         metric.remove("lockAttempt.acquireLoad");
         metric.remove("lockAttempt.lockedLatency");
+        metric.remove("lockAttempt.lockedMaxActiveLatency");
+        metric.remove("lockAttempt.lockedHz");
         metric.remove("lockAttempt.lockedLoad");
 
         assertEquals(expectedMetrics, new TreeMap<>(metric.values));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java
index 1e6fd5b86a6..09fb4d59443 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java
@@ -48,7 +48,7 @@ public class TestMetric implements Metric {
 
     double sumDoubleValues(String key, Context sumContext) {
         double sum = 0.0;
-        for(Context c : context.get(key)) {
+        for(Context c : context.getOrDefault(key, List.of())) {
             TestContext tc = (TestContext) c;
             if (tc.value instanceof Double && tc.properties.equals(((TestContext) sumContext).properties)) {
                 sum += (double) tc.value;
@@ -59,7 +59,7 @@ public class TestMetric implements Metric {
 
     double sumNumberValues(String key) {
         double sum = 0.0;
-        for(Context c : context.get(key)) {
+        for(Context c : context.getOrDefault(key, List.of())) {
             TestContext tc = (TestContext) c;
             sum += tc.value.doubleValue();
         }