Make system metrics testable.

4 files changed, 204 insertions, 53 deletions

diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java
index 8030e03b0ff..95f58813e6e 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java
@@ -8,6 +8,7 @@ import ai.vespa.metricsproxy.metric.model.MetricId;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -106,6 +107,23 @@ public class Metric {
         return new Metric(name, value, time, new LinkedHashMap<>(dimensions), getDescription());
     }
 
+    @Override
+    public boolean equals(Object obj) {
+        if ( ! (obj instanceof Metric) ) return false;
+        Metric rhs = (Metric) obj;
+        return name.equals(rhs.name)
+                && description.equals(rhs.description)
+                && value.equals(rhs.value)
+                && (time == rhs.time)
+                && Objects.equals(dimensions, rhs.dimensions)
+                && Objects.equals(consumers, rhs.consumers);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, description, value, dimensions, consumers, time);
+    }
+
     /**
      * @return the description of this metric
      */
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
index 9a688364b38..3f0ea4743ad 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
@@ -28,16 +28,16 @@ import java.util.logging.Logger;
 public class SystemPoller {
 
     private static final Logger log = Logger.getLogger(SystemPoller.class.getName());
+    private static final int memoryTypeVirtual = 0;
+    private static final int memoryTypeResident = 1;
 
     private final int pollingIntervalSecs;
     private final List<VespaService> services;
-
-    private final int memoryTypeVirtual = 0;
-    private final int memoryTypeResident = 1;
     private final Map<VespaService, Long> lastCpuJiffiesMetrics = new ConcurrentHashMap<>();
     private final Timer systemPollTimer;
+    private final GetJiffies jiffiesInterface;
 
-    private JiffiesAndCpus lastTotalCpuJiffies = null;
+    private JiffiesAndCpus lastTotalCpuJiffies;
 
     static class JiffiesAndCpus {
         final long jiffies;
@@ -56,11 +56,30 @@ public class SystemPoller {
                     : new JiffiesAndCpus();
         }
     }
+    interface GetJiffies {
+        JiffiesAndCpus getTotalSystemJiffies();
+        long getJiffies(VespaService service);
+    }
 
     public SystemPoller(List<VespaService> services, int pollingIntervalSecs) {
         this.services = services;
         this.pollingIntervalSecs = pollingIntervalSecs;
         systemPollTimer = new Timer("systemPollTimer", true);
+        jiffiesInterface = new GetJiffies() {
+            @Override
+            public JiffiesAndCpus getTotalSystemJiffies() {
+                return SystemPoller.getTotalSystemJiffies();
+            }
+
+            @Override
+            public long getJiffies(VespaService service) {
+                return SystemPoller.getPidJiffies(service);
+            }
+        };
+        lastTotalCpuJiffies = jiffiesInterface.getTotalSystemJiffies();
+        for (VespaService s : services) {
+            lastCpuJiffiesMetrics.put(s, jiffiesInterface.getJiffies(s));
+        }
     }
 
     void stop() {
@@ -74,7 +93,7 @@ public class SystemPoller {
      * @param service The instance to get memory usage for
      * @return array[0] = memoryResident, array[1] = memoryVirtual (kB units)
      */
-    long[] getMemoryUsage(VespaService service) {
+    static long[] getMemoryUsage(VespaService service) {
         long[] size = new long[2];
         BufferedReader br;
         int pid = service.getPid();
@@ -111,7 +130,6 @@ public class SystemPoller {
      */
     void poll() {
         long startTime = System.currentTimeMillis();
-        boolean someAlive = false;
 
         /* Don't do any work if there are no known services */
         if (services.isEmpty()) {
@@ -121,62 +139,59 @@ public class SystemPoller {
 
         log.log(Level.FINE, () -> "Monitoring system metrics for " + services.size() + " services");
 
-        JiffiesAndCpus sysJiffies = getTotalSystemJiffies();
-        for (VespaService s : services) {
-
+        boolean someAlive = services.stream().anyMatch(VespaService::isAlive);
+        lastTotalCpuJiffies = updateMetrics(lastTotalCpuJiffies, startTime/1000, jiffiesInterface, services, lastCpuJiffiesMetrics);
 
-            if(s.isAlive()) {
-                someAlive = true;
-            }
+        // If none of the services were alive, reschedule in a short time
+        if (!someAlive) {
+            reschedule(System.currentTimeMillis() - startTime);
+        } else {
+            schedule();
+        }
+    }
 
+    static JiffiesAndCpus updateMetrics(JiffiesAndCpus prevTotalJiffies, long timeStamp, GetJiffies getJiffies,
+                                        List<VespaService> services, Map<VespaService, Long> lastCpuJiffiesMetrics) {
+        JiffiesAndCpus sysJiffies = getJiffies.getTotalSystemJiffies();
+        JiffiesAndCpus sysJiffiesDiff = sysJiffies.diff(prevTotalJiffies);
+        for (VespaService s : services) {
             Metrics metrics = new Metrics();
             log.log(Level.FINE, () -> "Current size of system metrics for service  " + s + " is " + metrics.size());
 
             long[] size = getMemoryUsage(s);
             log.log(Level.FINE, () -> "Updating memory metric for service " + s);
 
-            long timeStamp = startTime / 1000;
             metrics.add(new Metric(MetricId.toMetricId("memory_virt"), size[memoryTypeVirtual], timeStamp));
             metrics.add(new Metric(MetricId.toMetricId("memory_rss"), size[memoryTypeResident], timeStamp));
 
-            long procJiffies = getPidJiffies(s);
-            if ((lastTotalCpuJiffies != null) && lastCpuJiffiesMetrics.containsKey(s)) {
-                long last = lastCpuJiffiesMetrics.get(s);
-                long diff = procJiffies - last;
+            long procJiffies = getJiffies.getJiffies(s);
+            long last = lastCpuJiffiesMetrics.get(s);
+            long diff = procJiffies - last;
 
-                if (diff >= 0) {
-                    JiffiesAndCpus sysJiffiesDiff = sysJiffies.diff(lastTotalCpuJiffies);
-                    metrics.add(new Metric(MetricId.toMetricId("cpu"), 100 * ((double) diff) / sysJiffiesDiff.normalizedJiffies(), timeStamp));
-                    metrics.add(new Metric(MetricId.toMetricId("cpu.util"), 100 * ((double) diff) / sysJiffiesDiff.jiffies, timeStamp));
-                }
+            if (diff >= 0) {
+                metrics.add(new Metric(MetricId.toMetricId("cpu"), 100 * ((double) diff) / sysJiffiesDiff.normalizedJiffies(), timeStamp));
+                metrics.add(new Metric(MetricId.toMetricId("cpu.util"), 100 * ((double) diff) / sysJiffiesDiff.jiffies, timeStamp));
             }
             lastCpuJiffiesMetrics.put(s, procJiffies);
             s.setSystemMetrics(metrics);
         }
-
-        lastTotalCpuJiffies = sysJiffies;
-
-        // If none of the services were alive, reschedule in a short time
-        if (!someAlive) {
-            reschedule(System.currentTimeMillis() - startTime);
-        } else {
-            schedule();
-        }
+        return sysJiffies;
     }
 
-    long getPidJiffies(VespaService service) {
-        BufferedReader in;
-        String line;
-        String[] elems;
+    static long getPidJiffies(VespaService service) {
         int pid = service.getPid();
-
         try {
-            in = new BufferedReader(new FileReader("/proc/" + pid + "/stat"));
+            BufferedReader in = new BufferedReader(new FileReader("/proc/" + pid + "/stat"));
+            return getPidJiffies(in);
         } catch (FileNotFoundException ex) {
             log.log(Level.FINE, () -> "Unable to find pid " + pid + " in proc directory, for service " + service.getInstanceName());
             service.setAlive(false);
             return 0;
         }
+    }
+    static long getPidJiffies(BufferedReader in) {
+        String line;
+        String[] elems;
 
         try {
             line = in.readLine();
@@ -192,19 +207,21 @@ public class SystemPoller {
         return Long.parseLong(elems[13]) + Long.parseLong(elems[14]);
     }
 
-    private JiffiesAndCpus getTotalSystemJiffies() {
-        BufferedReader in;
-        String line;
-        ArrayList<CpuJiffies> jiffies = new ArrayList<>();
-        CpuJiffies total = null;
-
+    private static JiffiesAndCpus getTotalSystemJiffies() {
         try {
-            in = new BufferedReader(new FileReader("/proc/stat"));
+            BufferedReader in = new BufferedReader(new FileReader("/proc/stat"));
+            return getTotalSystemJiffies(in);
         } catch (FileNotFoundException ex) {
             log.log(Level.SEVERE, "Unable to open stat file", ex);
             return new JiffiesAndCpus();
         }
+    }
+    static JiffiesAndCpus getTotalSystemJiffies(BufferedReader in) {
+        ArrayList<CpuJiffies> jiffies = new ArrayList<>();
+        CpuJiffies total = null;
+
         try {
+            String line;
             while ((line = in.readLine()) != null) {
                 if (line.startsWith("cpu ")) {
                     total = new CpuJiffies(line);
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
index c6e0b202985..8dd2b251a7f 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
@@ -10,6 +10,7 @@ import ai.vespa.metricsproxy.metric.model.ServiceId;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 
 /**
@@ -34,7 +35,7 @@ public class VespaService implements Comparable<VespaService> {
     private volatile boolean isAlive;
 
     // Used to keep the last polled system metrics for service
-    private Metrics systemMetrics;
+    private final AtomicReference<Metrics> systemMetrics = new AtomicReference<>();
 
     private final int statePort;
 
@@ -73,7 +74,7 @@ public class VespaService implements Comparable<VespaService> {
         this.configId = configId;
         this.statePort = statePort;
         this.dimensions = dimensions;
-        this.systemMetrics = new Metrics();
+        this.systemMetrics.set(new Metrics());
         this.isAlive = false;
         this.remoteMetricsFetcher = (this.statePort> 0) ? new RemoteMetricsFetcher(this, this.statePort) : new DummyMetricsFetcher(this);
         this.remoteHealthMetricFetcher = (this.statePort > 0) ? new RemoteHealthMetricFetcher(this, this.statePort) : new DummyHealthMetricFetcher(this);
@@ -128,8 +129,8 @@ public class VespaService implements Comparable<VespaService> {
      *
      * @return System metrics
      */
-    public synchronized Metrics getSystemMetrics() {
-        return this.systemMetrics;
+    public Metrics getSystemMetrics() {
+        return systemMetrics.get();
     }
 
     /**
@@ -217,8 +218,8 @@ public class VespaService implements Comparable<VespaService> {
         this.isAlive = alive;
     }
 
-    public synchronized void setSystemMetrics(Metrics systemMetrics) {
-        this.systemMetrics = systemMetrics;
+    public synchronized void setSystemMetrics(Metrics metrics) {
+        systemMetrics.set(metrics);
     }
 
 }
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java
index 86e67309e5a..64b19b5d91b 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java
@@ -1,12 +1,20 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.metricsproxy.service;
 
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.model.MetricId;
 import org.junit.Test;
 
+import java.io.BufferedReader;
+import java.io.StringReader;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 
 /**
@@ -14,23 +22,88 @@ import static org.junit.Assert.assertThat;
  */
 public class SystemPollerTest {
 
+    private static final String [] perProcStats = {
+            "131 (java) S 115 115 85 34816 115 1077952512 16819285 69606636 25448 8182 1310952 380185 256300 41075 20 0 69 0 39175 16913702912 2004355 18446744073709551615 94843481096192 94843481100104 140737227872512 0 0 0 0 4096 16796879 0 0 0 17 1 0 0 2 0 0 94843483200760 94843483201552 94843503726592 140737227877827 140737227878876 140737227878876 140737227882433 0\n",
+            "131 (java) S 115 115 85 34816 115 1077952512 16819325 69606636 25480 8182 1311491 380694 256300 41075 20 0 69 0 39175 16913702912 2004353 18446744073709551615 94843481096192 94843481100104 140737227872512 0 0 0 0 4096 16796879 0 0 0 17 1 0 0 2 0 0 94843483200760 94843483201552 94843503726592 140737227877827 140737227878876 140737227878876 140737227882433 0\n"
+    };
+    private static final String [] totalStats = {
+            String.join("\n",
+                    "cpu  2262158 26 745678 139770640 28866 0 21092 0 0 0",
+                    "cpu0 294842 1 83365 17489116 3671 0 2563 0 0 0",
+                    "cpu1 273560 3 107315 17416254 3952 0 2970 0 0 0",
+                    "cpu2 287921 1 100781 17444671 4149 0 2760 0 0 0",
+                    "cpu3 292114 5 100488 17451934 4137 0 2427 0 0 0",
+                    "cpu4 255834 1 91932 17502801 3267 0 2544 0 0 0",
+                    "cpu5 280032 0 84827 17509603 3561 0 2641 0 0 0",
+                    "cpu6 274831 12 96846 17459038 3189 0 2531 0 0 0",
+                    "cpu7 303021 0 80121 17497220 2937 0 2653 0 0 0",
+                    "intr 122075142 123 0 0 70 40110 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 986 0 1164882 0 18416 17418 0 3157177 1795096 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0",
+                    "ctxt 169897491",
+                    "btime 1634013028",
+                    "processes 214086",
+                    "procs_running 1",
+                    "procs_blocked 0",
+                    "softirq 32035299 0 4526992 8 5339012 1156237 0 37 10929205 0 10083808"),
+            String.join("\n",
+                    "cpu  2262521 26 746460 140037592 28870 0 21094 0 0 0",
+                    "cpu0 294872 1 83431 17522597 3672 0 2564 0 0 0",
+                    "cpu1 273619 3 107436 17449509 3952 0 2971 0 0 0",
+                    "cpu2 287976 1 100893 17477999 4149 0 2760 0 0 0",
+                    "cpu3 292150 5 100601 17485270 4137 0 2427 0 0 0",
+                    "cpu4 255880 1 92024 17536168 3267 0 2544 0 0 0",
+                    "cpu5 280088 0 84937 17542968 3564 0 2641 0 0 0",
+                    "cpu6 274877 12 96944 17492406 3189 0 2531 0 0 0",
+                    "cpu7 303058 0 80192 17530672 2937 0 2653 0 0 0",
+                    "intr 122204990 123 0 0 70 40202 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 988 0 1165034 0 18438 17442 0 3160615 1797273 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0",
+                    "ctxt 170103457",
+                    "btime 1634013028",
+                    "processes 214098",
+                    "procs_running 2",
+                    "procs_blocked 0",
+                    "softirq 32061301 0 4533497 8 5339359 1156384 0 37 10947075 0 10084941")
+    };
+    private static final long [] PER_PROC_JIFFIES = {1691137, 1692185};
+
     @Test
     public void testSystemPoller() {
         DummyService s = new DummyService(0, "id");
         List<VespaService> services = new ArrayList<>();
         services.add(s);
 
-        SystemPoller poller = new SystemPoller(services, 60*5);
         assertThat(s.isAlive(), is(false));
 
-        long n = poller.getPidJiffies(s);
+        long n = SystemPoller.getPidJiffies(s);
         assertThat(n, is(0L));
-        long[] memusage = poller.getMemoryUsage(s);
+        long[] memusage = SystemPoller.getMemoryUsage(s);
         assertThat(memusage[0], is(0L));
         assertThat(memusage[1], is(0L));
     }
 
     @Test
+    public
+    void testPerProcessJiffies() {
+        assertEquals(PER_PROC_JIFFIES[0], SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[0]))));
+        assertEquals(PER_PROC_JIFFIES[1], SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[1]))));
+    }
+
+    @Test
+    public
+    void testTotalJiffies() {
+        SystemPoller.JiffiesAndCpus first = SystemPoller.getTotalSystemJiffies(new BufferedReader(new StringReader(totalStats[0])));
+        SystemPoller.JiffiesAndCpus second = SystemPoller.getTotalSystemJiffies(new BufferedReader(new StringReader(totalStats[1])));
+        assertEquals(8, first.cpus);
+        assertEquals(first.cpus, second.cpus);
+        assertEquals(142828460L, first.jiffies);
+        assertEquals(143096563L, second.jiffies);
+        assertEquals(142828460L/8, first.normalizedJiffies());
+        assertEquals(143096563L/8, second.normalizedJiffies());
+        SystemPoller.JiffiesAndCpus diff = second.diff(first);
+        assertEquals(8, diff.cpus);
+        assertEquals(268103L, diff.jiffies);
+        assertEquals(268103L/8, diff.normalizedJiffies());
+    }
+
+    @Test
     public void testCPUJiffies() {
         String line = "cpu1  102180864 789 56766899 12800020140 1654757 0 0";
         CpuJiffies n = new CpuJiffies(line);
@@ -38,4 +111,46 @@ public class SystemPollerTest {
         assertThat(n.getTotalJiffies(), is(12960623449L));
     }
 
+    @Test
+    public void testMetricsComputation() {
+        SystemPoller.JiffiesAndCpus prev = SystemPoller.getTotalSystemJiffies(new BufferedReader(new StringReader(totalStats[0])));
+        Map<VespaService, Long> lastCpuJiffiesMetrics = new HashMap<>();
+        VespaService s1 = new VespaService("s1", "cfgId_1");
+        List<VespaService> services = List.of(s1);
+        lastCpuJiffiesMetrics.put(s1, SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[0]))));
+
+        SystemPoller.JiffiesAndCpus next = SystemPoller.updateMetrics(prev, 1,
+                new SystemPoller.GetJiffies() {
+                    @Override
+                    public SystemPoller.JiffiesAndCpus getTotalSystemJiffies() {
+                        return SystemPoller.getTotalSystemJiffies(new BufferedReader(new StringReader(totalStats[1])));
+                    }
+
+                    @Override
+                    public long getJiffies(VespaService service) {
+                        return SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[1])));
+                    }
+                },
+                services, lastCpuJiffiesMetrics);
+
+        assertEquals(8, prev.cpus);
+        assertEquals(prev.cpus, next.cpus);
+        assertEquals(142828460L, prev.jiffies);
+        assertEquals(143096563L, next.jiffies);
+        SystemPoller.JiffiesAndCpus diff = next.diff(prev);
+
+        Metrics m = s1.getSystemMetrics();
+        List<Metric> metricList = m.getMetrics();
+        assertEquals(4, metricList.size());
+        assertEquals(new Metric(MetricId.toMetricId("memory_virt"), 0L, 1), metricList.get(0));
+        assertEquals(new Metric(MetricId.toMetricId("memory_rss"), 0L, 1), metricList.get(1));
+        long diffProcJiffies = (PER_PROC_JIFFIES[1] - PER_PROC_JIFFIES[0]);
+        double expected = 100.0*diffProcJiffies/diff.normalizedJiffies();
+        assertEquals(3.1272380042969683, expected, 0.0000000000001);
+        double expected_util = 100.0*diffProcJiffies/diff.jiffies;
+        assertEquals(0.3908945442609743, expected_util, 0.0000000000001);
+        assertEquals(new Metric(MetricId.toMetricId("cpu"), expected, 1), metricList.get(2));
+        assertEquals(new Metric(MetricId.toMetricId("cpu.util"), expected_util, 1), metricList.get(3));
+    }
+
 }