Merge branch 'master' into freva/build-maven-dependencies

5 files changed, 103 insertions, 39 deletions

diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java
index 94e827b8a49..38837a4b12b 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java
@@ -1,29 +1,51 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.dockerapi;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.Objects;
 
 /**
  * @author stiankri
  */
 public class Container {
+    private static final DateTimeFormatter DOCKER_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSX");
     public final String hostname;
     public final DockerImage image;
     public final ContainerName name;
     public final State state;
     public final int pid;
+    public final String created;
 
     public Container(
             final String hostname,
             final DockerImage image,
             final ContainerName containerName,
             final State state,
-            final int pid) {
+            final int pid,
+            final String created) {
         this.hostname = hostname;
         this.image = image;
         this.name = containerName;
         this.state = state;
         this.pid = pid;
+        this.created = created;
+    }
+
+    // For testing only
+    public Container(
+            final String hostname,
+            final DockerImage image,
+            final ContainerName containerName,
+            final State state,
+            final int pid) {
+        this(hostname, image, containerName, state, pid, "2017-02-13T13:45:12.133713371Z");
+    }
+
+    public Instant getCreatedAsInstant() {
+        return LocalDateTime.parse(created, DOCKER_DATE_FORMAT).toInstant(ZoneOffset.UTC);
     }
 
     @Override
@@ -35,7 +57,8 @@ public class Container {
         return Objects.equals(hostname, other.hostname)
                 && Objects.equals(image, other.image)
                 && Objects.equals(name, other.name)
-                && Objects.equals(pid, other.pid);
+                && Objects.equals(pid, other.pid)
+                && Objects.equals(created, other.created);
     }
 
     @Override
@@ -51,6 +74,7 @@ public class Container {
                 + " name=" + name
                 + " state=" + state
                 + " pid=" + pid
+                + " created=" + created
                 + "}";
     }
 
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index fd4551d0f02..1ac5f62b130 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -364,7 +364,8 @@ public class DockerImpl implements Docker {
                                 new DockerImage(response.getConfig().getImage()),
                                 new ContainerName(decode(response.getName())),
                                 Container.State.valueOf(response.getState().getStatus().toUpperCase()),
-                                response.getState().getPid()
+                                response.getState().getPid(),
+                                response.getCreated()
                         ))
                 .map(Stream::of)
                 .orElse(Stream.empty());
@@ -506,6 +507,6 @@ public class DockerImpl implements Docker {
                 .add("host", HostName.getLocalhost())
                 .add("role", "docker").build();
 
-        numberOfDockerDaemonFails = metricReceiver.declareCounter(dimensions, "daemon.api_fails");
+        numberOfDockerDaemonFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "daemon.api_fails");
     }
 }
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
index cb5185e440c..7adcbe715ec 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java
@@ -6,12 +6,12 @@ import java.util.HashMap;
 import java.util.Map;
 
 /**
- * Each metric reported to secret agent has dimmensions.
+ * Each metric reported to secret agent has dimensions.
  *
  * @author valerijf
  */
 public class Dimensions {
-    public final Map<String, Object> dimensionsMap;
+    final Map<String, Object> dimensionsMap;
 
     private Dimensions(Map<String, Object> dimensionsMap) {
         this.dimensionsMap = dimensionsMap;
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
index 842ad722e5b..776211635fb 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java
@@ -7,21 +7,27 @@ import com.google.inject.Inject;
 import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.metrics.simple.Point;
 
+import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
- * Export metrics to both /state/v1/metrics and makes them available programatically.
+ * Export metrics to both /state/v1/metrics and makes them available programmatically.
+ * Each metric belongs to a yamas application
  *
  * @author valerijf
  */
-public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.DimensionMetrics> {
+public class MetricReceiverWrapper {
+    // Application names used
+    public static final String APPLICATION_DOCKER = "docker";
+    public static final String APPLICATION_HOST_LIFE = "host-life";
+
     private final static ObjectMapper objectMapper = new ObjectMapper();
     private final Object monitor = new Object();
-    private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new HashMap<>();
+    private final Map<String, ApplicationMetrics> applicationMetrics = new HashMap<>(); // key is application name
     private final MetricReceiver metricReceiver;
 
     @Inject
@@ -32,8 +38,9 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
     /**
      *  Declaring the same dimensions and name results in the same CounterWrapper instance (idempotent).
      */
-    public CounterWrapper declareCounter(Dimensions dimensions, String name) {
+    public CounterWrapper declareCounter(String application, Dimensions dimensions, String name) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             if (!metricsByDimensions.containsKey(dimensions)) metricsByDimensions.put(dimensions, new HashMap<>());
             if (!metricsByDimensions.get(dimensions).containsKey(name)) {
                 CounterWrapper counter = new CounterWrapper(metricReceiver.declareCounter(name, new Point(dimensions.dimensionsMap)));
@@ -47,8 +54,9 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
     /**
      *  Declaring the same dimensions and name results in the same GaugeWrapper instance (idempotent).
      */
-    public GaugeWrapper declareGauge(Dimensions dimensions, String name) {
+    public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             if (!metricsByDimensions.containsKey(dimensions))
                 metricsByDimensions.put(dimensions, new HashMap<>());
             if (!metricsByDimensions.get(dimensions).containsKey(name)) {
@@ -62,38 +70,50 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
 
     public void unsetMetricsForContainer(String hostname) {
         synchronized (monitor) {
-            Iterator<Dimensions> dimensionsIterator = metricsByDimensions.keySet().iterator();
-            while (dimensionsIterator.hasNext()) {
-                Dimensions dimension = dimensionsIterator.next();
-                if (dimension.dimensionsMap.containsKey("host") && dimension.dimensionsMap.get("host").equals(hostname)) {
-                    dimensionsIterator.remove();
-                }
-            }
+            applicationMetrics.values()
+                              .forEach(m -> m.metricsByDimensions.keySet()
+                                                                 .removeIf(d -> d.dimensionsMap.containsKey("host") &&
+                                                                         d.dimensionsMap.get("host").equals(hostname)));
         }
     }
 
+    public List<DimensionMetrics> getMetrics(String application) {
+        synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
+            return metricsByDimensions.entrySet()
+                                     .stream()
+                                     .map(entry -> new DimensionMetrics(application, entry.getKey(), entry.getValue()))
+                                     .collect(Collectors.toList());
+        }
+    }
 
-    @Override
-    public Iterator<DimensionMetrics> iterator() {
+    public List<DimensionMetrics> getAllMetrics() {
         synchronized (monitor) {
-            return metricsByDimensions.entrySet().stream().map(entry ->
-                    new DimensionMetrics(entry.getKey(), entry.getValue())).iterator();
+            List<DimensionMetrics> dimensionMetrics = new ArrayList<>();
+            applicationMetrics.entrySet()
+                              .forEach(e -> e.getValue().metricsByDimensions().entrySet().stream()
+                                             .map(entry -> new DimensionMetrics(e.getKey(), entry.getKey(), entry.getValue()))
+                                             .map(dimensionMetrics::add));
+            return dimensionMetrics;
         }
     }
 
     // For testing
-    Map<String, Number> getMetricsForDimension(Dimensions dimensions) {
+    Map<String, Number> getMetricsForDimension(String application, Dimensions dimensions) {
         synchronized (monitor) {
+            Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application);
             return metricsByDimensions.get(dimensions).entrySet().stream().collect(
                     Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
         }
     }
 
     public class DimensionMetrics {
+        private final String application;
         private final Dimensions dimensions;
         private final Map<String, Object> metrics;
 
-        DimensionMetrics(Dimensions dimensions, Map<String, MetricValue> metricValues) {
+        DimensionMetrics(String application, Dimensions dimensions, Map<String, MetricValue> metricValues) {
+            this.application = application;
             this.dimensions = dimensions;
             this.metrics = metricValues.entrySet().stream().collect(
                     Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue()));
@@ -106,7 +126,7 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
             routingYamas.put("namespaces", new String[]{"Vespa"});
 
             Map<String, Object> report = new LinkedHashMap<>();
-            report.put("application", "docker");
+            report.put("application", application);
             report.put("timestamp", System.currentTimeMillis() / 1000);
             report.put("dimensions", dimensions.dimensionsMap);
             report.put("metrics", metrics);
@@ -115,4 +135,21 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim
             return objectMapper.writeValueAsString(report);
         }
     }
+
+    private Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application) {
+        if (! applicationMetrics.containsKey(application)) {
+            ApplicationMetrics metrics = new ApplicationMetrics();
+            applicationMetrics.put(application, metrics);
+        }
+        return applicationMetrics.get(application).metricsByDimensions();
+    }
+
+    // Application is yamas application, not Vespa application
+    private static class ApplicationMetrics {
+        private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>();
+
+        Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() {
+            return metricsByDimensions;
+        }
+    }
 }
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
index 82de7c8c8d3..1cbc83fb9af 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java
@@ -14,24 +14,26 @@ import static org.junit.Assert.assertTrue;
  */
 public class MetricReceiverWrapperTest {
     private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
+    private static final String applicationDocker = MetricReceiverWrapper.APPLICATION_DOCKER;
 
     @Test
     public void testDefaultValue() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        metricReceiver.declareCounter(hostDimension, "some.name");
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("some.name"), 0L);
+        metricReceiver.declareCounter(applicationDocker, hostDimension, "some.name");
+
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("some.name"), 0L);
     }
 
     @Test
     public void testSimpleIncrementMetric() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        CounterWrapper counter = metricReceiver.declareCounter(hostDimension, "a_counter.value");
+        CounterWrapper counter = metricReceiver.declareCounter(applicationDocker, hostDimension, "a_counter.value");
 
         counter.add(5);
         counter.add(8);
 
-        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(hostDimension);
+        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension);
         assertTrue("Expected only 1 metric value to be set", latestMetrics.size() == 1);
         assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8
     }
@@ -39,12 +41,12 @@ public class MetricReceiverWrapperTest {
     @Test
     public void testSimpleGauge() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
 
         gauge.sample(42);
         gauge.sample(-342.23);
 
-        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(hostDimension);
+        Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension);
         assertTrue("Expected only 1 metric value to be set", latestMetrics.size() == 1);
         assertEquals(latestMetrics.get("test.gauge"), -342.23);
     }
@@ -52,29 +54,29 @@ public class MetricReceiverWrapperTest {
     @Test
     public void testRedeclaringSameGauge() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
         gauge.sample(42);
 
         // Same as hostDimension, but new instance.
         Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build();
-        GaugeWrapper newGauge = metricReceiver.declareGauge(newDimension, "test.gauge");
+        GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge");
         newGauge.sample(56);
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("test.gauge"), 56.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 56.);
     }
 
     @Test
     public void testSameMetricNameButDifferentDimensions() {
         MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
-        GaugeWrapper gauge = metricReceiver.declareGauge(hostDimension, "test.gauge");
+        GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge");
         gauge.sample(42);
 
         // Not the same as hostDimension.
         Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build();
-        GaugeWrapper newGauge = metricReceiver.declareGauge(newDimension, "test.gauge");
+        GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge");
         newGauge.sample(56);
 
-        assertEquals(metricReceiver.getMetricsForDimension(hostDimension).get("test.gauge"), 42.);
-        assertEquals(metricReceiver.getMetricsForDimension(newDimension).get("test.gauge"), 56.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 42.);
+        assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, newDimension).get("test.gauge"), 56.);
     }
 }