Rename throttled_time to throttled_cpu_time and add new throttled_time

3 files changed, 36 insertions, 20 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 5ca02f82aa5..3a9eae45607 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -546,7 +546,8 @@ public class NodeAgentImpl implements NodeAgent {
         final double allocatedCpuRatio = node.vcpus() / totalNumCpuCores;
         double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio;
         double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio;
-        Long cpuThrottledTime = lastCpuMetric.getThrottledTime();
+        double cpuThrottledTime = lastCpuMetric.getThrottledTime();
+        double cpuThrottledCpuTime = lastCpuMetric.getThrottledCpuTime();
 
         long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache;
         double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes;
@@ -562,7 +563,8 @@ public class NodeAgentImpl implements NodeAgent {
                 .withMetric("mem_total.util", 100 * memoryTotalUsageRatio)
                 .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated)
                 .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated)
-                .withMetric("cpu.throttled_time.ns", cpuThrottledTime)
+                .withMetric("cpu.throttled_time", cpuThrottledTime)
+                .withMetric("cpu.throttled_cpu_time", cpuThrottledCpuTime)
                 .withMetric("cpu.vcpus", node.vcpus())
                 .withMetric("disk.limit", diskTotalBytes);
 
@@ -620,26 +622,35 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     class CpuUsageReporter {
+        private static final double BILLION = 1_000_000_000d;
         private long containerKernelUsage = 0;
         private long totalContainerUsage = 0;
         private long totalSystemUsage = 0;
         private long throttledTime = 0;
+        private long throttlingActivePeriods = 0;
+        private long throttledPeriods = 0;
 
         private long deltaContainerKernelUsage;
         private long deltaContainerUsage;
         private long deltaSystemUsage;
         private long deltaThrottledTime;
+        private long deltaThrottlingActivePeriods;
+        private long deltaThrottledPeriods;
 
         private void updateCpuDeltas(ContainerStats.CpuStats cpuStats) {
-            deltaSystemUsage = this.totalSystemUsage == 0 ? 0 : (cpuStats.getSystemCpuUsage() - this.totalSystemUsage);
-            deltaContainerUsage = cpuStats.getTotalUsage() - this.totalContainerUsage;
-            deltaContainerKernelUsage = cpuStats.getUsageInKernelMode() - this.containerKernelUsage;
-            deltaThrottledTime = cpuStats.getThrottledTime() - this.throttledTime;
-
-            this.totalSystemUsage = cpuStats.getSystemCpuUsage();
-            this.totalContainerUsage = cpuStats.getTotalUsage();
-            this.containerKernelUsage = cpuStats.getUsageInKernelMode();
-            this.throttledTime = cpuStats.getThrottledTime();
+            deltaSystemUsage = totalSystemUsage == 0 ? 0 : (cpuStats.getSystemCpuUsage() - totalSystemUsage);
+            deltaContainerUsage = cpuStats.getTotalUsage() - totalContainerUsage;
+            deltaContainerKernelUsage = cpuStats.getUsageInKernelMode() - containerKernelUsage;
+            deltaThrottledTime = cpuStats.getThrottledTime() - throttledTime;
+            deltaThrottlingActivePeriods = cpuStats.getThrottlingActivePeriods() - throttlingActivePeriods;
+            deltaThrottledPeriods = cpuStats.getThrottledPeriods() - throttledPeriods;
+
+            totalSystemUsage = cpuStats.getSystemCpuUsage();
+            totalContainerUsage = cpuStats.getTotalUsage();
+            containerKernelUsage = cpuStats.getUsageInKernelMode();
+            throttledTime = cpuStats.getThrottledTime();
+            throttlingActivePeriods = cpuStats.getThrottlingActivePeriods();
+            throttledPeriods = cpuStats.getThrottledPeriods();
         }
 
         /**
@@ -655,8 +666,12 @@ public class NodeAgentImpl implements NodeAgent {
             return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerKernelUsage / deltaSystemUsage;
         }
 
-        Long getThrottledTime() {
-            return deltaSystemUsage == 0 ? null : deltaThrottledTime;
+        double getThrottledTime() {
+            return deltaSystemUsage == 0 ? Double.NaN : 60d * deltaThrottledPeriods / deltaThrottlingActivePeriods;
+        }
+
+        double getThrottledCpuTime() {
+            return deltaSystemUsage == 0 ? Double.NaN : deltaThrottledTime / BILLION;
         }
     }
 
diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json
index ff4a2fde943..5b42d9a2428 100644
--- a/node-admin/src/test/resources/docker.stats.json
+++ b/node-admin/src/test/resources/docker.stats.json
@@ -18,9 +18,9 @@
     },
     "system_cpu_usage":5876874910000000,
     "throttling_data":{
-      "periods":3212,
-      "throttled_periods":322,
-      "throttled_time":4490
+      "periods":820694,
+      "throttled_periods":177731,
+      "throttled_time":81891944744550
     }
   },
   "cpu_stats":{
@@ -41,9 +41,9 @@
     },
     "system_cpu_usage":5876882680000000,
     "throttling_data":{
-      "periods":3242,
-      "throttled_periods":332,
-      "throttled_time":4523
+      "periods":821264,
+      "throttled_periods":178201,
+      "throttled_time":82181944744550
     }
   },
   "memory_stats":{
diff --git a/node-admin/src/test/resources/expected.container.system.metrics.txt b/node-admin/src/test/resources/expected.container.system.metrics.txt
index 5400edfec65..4876466d8ca 100644
--- a/node-admin/src/test/resources/expected.container.system.metrics.txt
+++ b/node-admin/src/test/resources/expected.container.system.metrics.txt
@@ -10,7 +10,8 @@ s:
     },
     "metrics": {
         "cpu.sys.util": 3.402,
-        "cpu.throttled_time.ns": 33,
+        "cpu.throttled_cpu_time": 290.0,
+        "cpu.throttled_time": 49.473,
         "cpu.util": 5.4,
         "cpu.vcpus": 2.0,
         "disk.limit": 250000000000,