summaryrefslogtreecommitdiffstats
path: root/metrics-proxy
diff options
context:
space:
mode:
authorgjoranv <gv@verizonmedia.com>2019-05-07 15:58:04 +0200
committergjoranv <gv@verizonmedia.com>2019-05-10 10:56:39 +0200
commitcf508533e0def83dcc4702d4ff83ad07e31f5b75 (patch)
treee173872e05ddc43aef1ce533e284f6cf1b18a9da /metrics-proxy
parent1e98247ac92f391bf8af18627354f2374255f32b (diff)
New metrics-proxy
Diffstat (limited to 'metrics-proxy')
-rw-r--r--metrics-proxy/CMakeLists.txt2
-rw-r--r--metrics-proxy/OWNERS1
-rw-r--r--metrics-proxy/README1
-rw-r--r--metrics-proxy/pom.xml157
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java80
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java160
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java307
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java8
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java45
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java99
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java56
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java124
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java111
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java71
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java30
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java30
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java8
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java38
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java38
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java39
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java182
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java39
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java130
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java80
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java130
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java8
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java62
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java213
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java176
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java43
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java34
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java30
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java94
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java77
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java129
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java14
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java259
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java33
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java216
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java123
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java8
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def5
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/consumers.def10
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/monitoring.def8
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def5
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def4
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/vespa-services.def10
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java27
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java244
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java73
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java99
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java111
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java63
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java94
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java144
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java97
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java208
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java104
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java61
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java67
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java36
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java90
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java50
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java56
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java45
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java77
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java42
-rw-r--r--metrics-proxy/src/test/resources/health-check-failed.response.json7
-rw-r--r--metrics-proxy/src/test/resources/health-check.response.json6
-rw-r--r--metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json281
-rw-r--r--metrics-proxy/src/test/resources/metrics-state.json42
-rw-r--r--metrics-proxy/src/test/resources/metrics-storage-simple.json38
-rw-r--r--metrics-proxy/src/test/resources/rpc-json-output-check.json1
-rw-r--r--metrics-proxy/src/test/resources/yamas-array-no-routing.json19
-rw-r--r--metrics-proxy/src/test/resources/yamas-array.json26
75 files changed, 5735 insertions, 0 deletions
diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt
new file mode 100644
index 00000000000..28b178fe636
--- /dev/null
+++ b/metrics-proxy/CMakeLists.txt
@@ -0,0 +1,2 @@
+# Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_fat_java_artifact(metrics-proxy)
diff --git a/metrics-proxy/OWNERS b/metrics-proxy/OWNERS
new file mode 100644
index 00000000000..3b2ba1ede81
--- /dev/null
+++ b/metrics-proxy/OWNERS
@@ -0,0 +1 @@
+gjoranv
diff --git a/metrics-proxy/README b/metrics-proxy/README
new file mode 100644
index 00000000000..4008cb1aa5d
--- /dev/null
+++ b/metrics-proxy/README
@@ -0,0 +1 @@
+The metrics proxy provides a single point of access for metrics from all Vespa services.
diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
new file mode 100644
index 00000000000..88c250f31bd
--- /dev/null
+++ b/metrics-proxy/pom.xml
@@ -0,0 +1,157 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>7-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>metrics-proxy</artifactId>
+ <packaging>container-plugin</packaging>
+ <version>7-SNAPSHOT</version>
+ <dependencies>
+
+ <!-- provided -->
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>annotations</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>component</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-lib</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-di</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jrt</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespalog</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>yolean</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- compile scope -->
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>http-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
+
+ <!-- test scope -->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-class-plugin</artifactId>
+ <version>${project.version}</version>
+ <executions>
+ <execution>
+ <id>config-gen</id>
+ <goals>
+ <goal>config-gen</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <packagePrefix>ai.vespa.</packagePrefix>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <!-- Only added to make IntelliJ use correct language level -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java
new file mode 100644
index 00000000000..564de0806ca
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.core;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collector;
+
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static com.yahoo.stream.CustomCollectors.toLinkedMap;
+import static java.util.Collections.unmodifiableSet;
+import static java.util.stream.Collectors.collectingAndThen;
+
+/**
+ * Contains metrics consumers and their metrics, and mappings between these.
+ * All collections are final and immutable.
+ *
+ * @author gjoranv
+ */
+public class MetricsConsumers {
+
+ // All metrics for each consumer.
+ private final Map<ConsumerId, List<Consumer.Metric>> consumerMetrics;
+
+ // All consumers for each metric (more useful than the opposite map).
+ private final Map<Consumer.Metric, List<ConsumerId>> consumersByMetric;
+
+ public MetricsConsumers(ConsumersConfig config) {
+ consumerMetrics = config.consumer().stream().collect(
+ toUnmodifiableLinkedMap(consumer -> toConsumerId(consumer.name()), Consumer::metric));
+
+ consumersByMetric = createConsumersByMetric(consumerMetrics);
+ }
+
+ /**
+ * @param consumer The consumer
+ * @return The metrics for the given consumer.
+ */
+ public List<Consumer.Metric> getMetricDefinitions(ConsumerId consumer) {
+ return consumerMetrics.get(consumer);
+ }
+
+ public Map<Consumer.Metric, List<ConsumerId>> getConsumersByMetric() {
+ return consumersByMetric;
+ }
+
+ public Set<ConsumerId> getAllConsumers() {
+ return unmodifiableSet(consumerMetrics.keySet());
+ }
+
+ /**
+ * Helper function to create mapping from metric to consumers.
+ * TODO: consider reversing the mapping in metrics-consumers.def instead: metric{}.consumer[]
+ */
+ private static Map<Consumer.Metric, List<ConsumerId>>
+ createConsumersByMetric(Map<ConsumerId, List<Consumer.Metric>> metricsByConsumer) {
+ Map<Consumer.Metric, List<ConsumerId>> consumersByMetric = new LinkedHashMap<>();
+ metricsByConsumer.forEach(
+ (consumer, metrics) -> metrics.forEach(
+ metric -> consumersByMetric.computeIfAbsent(metric, unused -> new ArrayList<>())
+ .add(consumer)));
+ return Collections.unmodifiableMap(consumersByMetric);
+ }
+
+ public static <T, K, U> Collector<T, ?, Map<K, U>> toUnmodifiableLinkedMap(Function<? super T, ? extends K> keyMapper,
+ Function<? super T, ? extends U> valueMapper) {
+ return collectingAndThen(toLinkedMap(keyMapper, valueMapper), Collections::unmodifiableMap);
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
new file mode 100644
index 00000000000..fe823c72127
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.core;
+
+import ai.vespa.metricsproxy.metric.ExternalMetrics;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.yahoo.component.Vtag;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static com.yahoo.log.LogLevel.DEBUG;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Retrieves metrics and performs necessary conversions and additions of metadata.
+ *
+ * @author gjoranv
+ */
+public class MetricsManager {
+ private static Logger log = Logger.getLogger(MetricsManager.class.getName());
+
+ static final DimensionId VESPA_VERSION = toDimensionId("vespaVersion");
+
+ private final VespaServices vespaServices;
+ private final VespaMetrics vespaMetrics;
+ private final ExternalMetrics externalMetrics;
+ private final ApplicationDimensions applicationDimensions;
+ private final NodeDimensions nodeDimensions;
+
+ private volatile Map<DimensionId, String> extraDimensions = new HashMap<>();
+
+ public MetricsManager(VespaServices vespaServices,
+ VespaMetrics vespaMetrics,
+ ExternalMetrics externalMetrics,
+ ApplicationDimensions applicationDimensions,
+ NodeDimensions nodeDimensions) {
+ this.vespaServices = vespaServices;
+ this.vespaMetrics = vespaMetrics;
+ this.externalMetrics = externalMetrics;
+ this.applicationDimensions = applicationDimensions;
+ this.nodeDimensions = nodeDimensions;
+ }
+
+ /**
+ * Returns all metrics for the given service that are whitelisted for the given consumer.
+ */
+ public String getMetricNamesForServiceAndConsumer(String service, ConsumerId consumer) {
+ return vespaMetrics.getMetricNames(vespaServices.getMonitoringServices(service), consumer);
+ }
+
+ public String getMetricsByConfigId(String configId) {
+ List<VespaService> services = vespaServices.getInstancesById(configId);
+ vespaServices.updateServices(services);
+
+ return vespaMetrics.getMetricsAsString(services);
+ }
+
+ /**
+ * Returns the metrics for the given services. The empty list is returned if no services are given.
+ *
+ * @param services The services to retrieve metrics for.
+ * @return Metrics for all matching services.
+ */
+ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) {
+ if (services.isEmpty()) return Collections.emptyList();
+ vespaServices.updateServices(services);
+
+ List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services);
+ log.log(DEBUG, () -> "Got " + result.size() + " metrics packets for vespa services.");
+
+ List<MetricsPacket.Builder> externalPackets = externalMetrics.getMetrics().stream()
+ .filter(MetricsPacket.Builder::hasMetrics)
+ .collect(toList());
+ log.log(DEBUG, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics.");
+
+ result.addAll(externalPackets);
+
+ return result.stream()
+ .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions()))
+ .map(builder -> builder.putDimensionsIfAbsent(extraDimensions))
+ .map(builder -> adjustTimestamp(builder, startTime))
+ .map(MetricsPacket.Builder::build)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Returns a merged map of all global dimensions.
+ */
+ private Map<DimensionId, String> getGlobalDimensions() {
+ Map<DimensionId, String> globalDimensions = new LinkedHashMap<>(applicationDimensions.getDimensions());
+ globalDimensions.putAll(nodeDimensions.getDimensions());
+ globalDimensions.put(VESPA_VERSION, Vtag.currentVersion.toFullString());
+ return globalDimensions;
+ }
+
+ /**
+ * If the metrics packet is less than one minute newer or older than the given startTime,
+ * set its timestamp to the given startTime. This is done to ensure that metrics retrieved
+ * from different sources for this invocation get the same timestamp, and a timestamp as close
+ * as possible to the invocation from the external metrics retrieving client. The assumption
+ * is that the client requests metrics periodically every minute.
+ * <p>
+ * However, if the timestamp of the packet is too far off in time, we don't adjust it because
+ * we would otherwise be masking a real problem with retrieving the metrics.
+ */
+ static MetricsPacket.Builder adjustTimestamp(MetricsPacket.Builder builder, Instant startTime) {
+ Duration age = Duration.between(startTime, builder.getTimestamp());
+ if (age.abs().minusMinutes(1).isNegative())
+ builder.timestamp(startTime.getEpochSecond());
+ return builder;
+ }
+
+ /**
+ * Returns the health metrics for the given services. The empty list is returned if no services are given.
+ *
+ * @param services The services to retrieve health metrics for.
+ * @return Health metrics for all matching services.
+ */
+ public List<MetricsPacket> getHealthMetrics(List<VespaService> services) {
+ if (services.isEmpty()) return Collections.emptyList();
+ vespaServices.updateServices(services);
+
+ // TODO: Add global dimensions to health metrics?
+ return vespaMetrics.getHealthMetrics(services);
+ }
+
+ public void setExtraMetrics(List<MetricsPacket.Builder> packets) {
+ extraDimensions = extractConfigserverDimensions(packets);
+ externalMetrics.setExtraMetrics(packets);
+ }
+
+ /**
+ * Returns a space separated list of all distinct service names.
+ */
+ public String getAllVespaServices() {
+ return vespaServices.getVespaServices().stream()
+ .map(VespaService::getServiceName)
+ .distinct()
+ .collect(Collectors.joining(" "));
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
new file mode 100644
index 00000000000..becfd9a54ce
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
@@ -0,0 +1,307 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.core;
+
+
+import ai.vespa.metricsproxy.metric.AggregationKey;
+import ai.vespa.metricsproxy.metric.HealthMetric;
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.MetricsFormatter;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.yahoo.log.LogLevel.DEBUG;
+
+/**
+ * @author Unknown
+ * @author gjoranv
+ */
+public class VespaMetrics {
+ private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName());
+
+ // MUST be the same as the constant defined in config-model
+ public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa");
+
+ public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype");
+ public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance");
+
+ private static final Set<ConsumerId> DEFAULT_CONSUMERS = Collections.singleton(VESPA_CONSUMER_ID);
+
+ private final MetricsConsumers metricsConsumers;
+
+ private static final MetricsFormatter formatter = new MetricsFormatter(false, false);
+
+ public VespaMetrics(MetricsConsumers metricsConsumers, VespaServices vespaServices) {
+ this.metricsConsumers = metricsConsumers;
+ }
+
+ public List<MetricsPacket> getHealthMetrics(List<VespaService> services) {
+ List<MetricsPacket> result = new ArrayList<>();
+ for (VespaService s : services) {
+ HealthMetric h = s.getHealth();
+ MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(s.getMonitoringName()))
+ .statusCode(h.isOk() ? 0 : 1)
+ .statusMessage(h.getMessage())
+ .putDimension(METRIC_TYPE_DIMENSION_ID, "health")
+ .putDimension(INSTANCE_DIMENSION_ID, s.getInstanceName());
+
+ result.add(builder.build());
+ }
+
+ return result;
+ }
+
+ /**
+ * @param services The services to get metrics for
+ * @return A list of metrics packet builders (to allow modification by the caller).
+ */
+ public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) {
+ List<MetricsPacket.Builder> metricsPackets = new ArrayList<>();
+
+ log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size());
+
+ Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric();
+
+ for (VespaService service : services) {
+ // One metrics packet for system metrics
+ Optional<MetricsPacket.Builder> systemCheck = getSystemMetrics(service);
+ systemCheck.ifPresent(metricsPackets::add);
+
+ // One metrics packet per set of metrics that share the same dimensions+consumers
+ // TODO: Move aggregation into MetricsPacket itself?
+ Metrics serviceMetrics = getServiceMetrics(service, consumersByMetric);
+ Map<AggregationKey, List<Metric>> aggregatedMetrics =
+ aggregateMetrics(service.getDimensions(), serviceMetrics);
+
+ aggregatedMetrics.forEach((aggregationKey, metrics) -> {
+ MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName()))
+ .putMetrics(metrics)
+ .putDimension(METRIC_TYPE_DIMENSION_ID, "standard")
+ .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName())
+ .putDimensions(aggregationKey.getDimensions());
+ setMetaInfo(builder, serviceMetrics.getTimeStamp());
+ builder.addConsumers(aggregationKey.getConsumers());
+ metricsPackets.add(builder);
+ });
+ }
+
+ return metricsPackets;
+ }
+
+ /**
+ * Returns the metrics to output for the given service, with updated timestamp
+ * In order to include a metric, it must exist in the given map of metric to consumers.
+ * Each returned metric will contain a collection of consumers that it should be routed to.
+ */
+ private Metrics getServiceMetrics(VespaService service, Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) {
+ Metrics serviceMetrics = new Metrics();
+ Metrics allServiceMetrics = service.getMetrics();
+ serviceMetrics.setTimeStamp(getMostRecentTimestamp(allServiceMetrics));
+ for (Metric candidate : allServiceMetrics.getMetrics()) {
+ getConfiguredMetrics(candidate.getName(), consumersByMetric.keySet()).forEach(
+ configuredMetric -> serviceMetrics.add(
+ metricWithConfigProperties(candidate, configuredMetric, consumersByMetric)));
+ }
+ return serviceMetrics;
+ }
+
+ private Map<DimensionId, String> extractDimensions(Map<DimensionId, String> dimensions, List<ConsumersConfig.Consumer.Metric.Dimension> configuredDimensions) {
+ if ( ! configuredDimensions.isEmpty()) {
+ Map<DimensionId, String> dims = new HashMap<>(dimensions);
+ configuredDimensions.forEach(d -> dims.put(toDimensionId(d.key()), d.value()));
+ dimensions = Collections.unmodifiableMap(dims);
+ }
+ return dimensions;
+ }
+
+ private Set<ConsumerId> extractConsumers(List<ConsumerId> configuredConsumers) {
+ Set<ConsumerId> consumers = Collections.emptySet();
+ if (configuredConsumers != null) {
+ if ( configuredConsumers.size() == 1) {
+ consumers = Collections.singleton(configuredConsumers.get(0));
+ } else if (configuredConsumers.size() > 1){
+ consumers = new HashSet<>();
+ consumers.addAll(configuredConsumers);
+ consumers = Collections.unmodifiableSet(consumers);
+ }
+ }
+ return consumers;
+ }
+
+ private Metric metricWithConfigProperties(Metric candidate,
+ ConsumersConfig.Consumer.Metric configuredMetric,
+ Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) {
+ Metric metric = candidate.clone();
+ metric.setDimensions(extractDimensions(candidate.getDimensions(), configuredMetric.dimension()));
+ metric.setConsumers(extractConsumers(consumersByMetric.get(configuredMetric)));
+
+ if (!isNullOrEmpty(configuredMetric.outputname()))
+ metric.setName(configuredMetric.outputname());
+ return metric;
+ }
+
+ /**
+ * Returns all configured metrics (for any consumer) that have the given id as 'name'.
+ */
+ private static Set<ConsumersConfig.Consumer.Metric> getConfiguredMetrics(String id,
+ Set<ConsumersConfig.Consumer.Metric> configuredMetrics) {
+ return configuredMetrics.stream()
+ .filter(m -> m.name().equals(id))
+ .collect(Collectors.toSet());
+ }
+
+ private Optional<MetricsPacket.Builder> getSystemMetrics(VespaService service) {
+ Metrics systemMetrics = service.getSystemMetrics();
+ if (systemMetrics.size() == 0) return Optional.empty();
+
+ MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName()));
+ setMetaInfo(builder, systemMetrics.getTimeStamp());
+
+ builder.putDimension(METRIC_TYPE_DIMENSION_ID, "system")
+ .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName())
+ .putDimensions(service.getDimensions())
+ .putMetrics(systemMetrics.getMetrics());
+
+ builder.addConsumers(metricsConsumers.getAllConsumers());
+ return Optional.of(builder);
+ }
+
+ private long getMostRecentTimestamp(Metrics metrics) {
+ long mostRecentTimestamp = 0L;
+ for (Metric metric : metrics.getMetrics()) {
+ if (metric.getTimeStamp() > mostRecentTimestamp) {
+ mostRecentTimestamp = metric.getTimeStamp();
+ }
+ }
+ return mostRecentTimestamp;
+ }
+
+ private Map<AggregationKey, List<Metric>> aggregateMetrics(Map<DimensionId, String> serviceDimensions,
+ Metrics metrics) {
+ Map<AggregationKey, List<Metric>> aggregatedMetrics = new HashMap<>();
+
+ for (Metric metric : metrics.getMetrics() ) {
+ Map<DimensionId, String> mergedDimensions = new LinkedHashMap<>();
+ mergedDimensions.putAll(metric.getDimensions());
+ mergedDimensions.putAll(serviceDimensions);
+ AggregationKey aggregationKey = new AggregationKey(mergedDimensions, metric.getConsumers());
+
+ if (aggregatedMetrics.containsKey(aggregationKey)) {
+ aggregatedMetrics.get(aggregationKey).add(metric);
+ } else {
+ List<Metric> ml = new ArrayList<>();
+ ml.add(metric);
+ aggregatedMetrics.put(aggregationKey, ml);
+ }
+ }
+ return aggregatedMetrics;
+ }
+
+ private List<ConsumersConfig.Consumer.Metric> getMetricDefinitions(ConsumerId consumer) {
+ if (metricsConsumers == null) return Collections.emptyList();
+
+ List<ConsumersConfig.Consumer.Metric> definitions = metricsConsumers.getMetricDefinitions(consumer);
+ return definitions == null ? Collections.emptyList() : definitions;
+ }
+
+ private static void setMetaInfo(MetricsPacket.Builder builder, long timestamp) {
+ builder.timestamp(timestamp)
+ .statusCode(0)
+ .statusMessage("Data collected successfully");
+ }
+
+ /**
+ * Returns a string representation of metrics for the given services;
+ * a space separated list of key=value.
+ */
+ public String getMetricsAsString(List<VespaService> services) {
+ StringBuilder b = new StringBuilder();
+ for (VespaService s : services) {
+ for (Metric metric : s.getMetrics().getMetrics()) {
+ String key = metric.getName();
+ String alias = key;
+
+ boolean isForwarded = false;
+ for (ConsumersConfig.Consumer.Metric metricConsumer : getMetricDefinitions(VESPA_CONSUMER_ID)) {
+ if (metricConsumer.name().equals(key)) {
+ alias = metricConsumer.outputname();
+ isForwarded = true;
+ }
+ }
+ if (isForwarded) {
+ b.append(formatter.format(s, alias, metric.getValue()))
+ .append(" ");
+ }
+ }
+ }
+ return b.toString();
+ }
+
+ /**
+ * Get all metric names for the given services
+ *
+ * @return String representation
+ */
+ public String getMetricNames(List<VespaService> services, ConsumerId consumer) {
+ StringBuilder bufferOn = new StringBuilder();
+ StringBuilder bufferOff = new StringBuilder();
+ for (VespaService s : services) {
+
+ for (Metric m : s.getMetrics().getMetrics()) {
+ String description = m.getDescription();
+ String alias = "";
+ boolean isForwarded = false;
+
+ for (ConsumersConfig.Consumer.Metric metric : getMetricDefinitions(consumer)) {
+ if (metric.name().equals(m.getName())) {
+ alias = metric.outputname();
+ isForwarded = true;
+ if (description.isEmpty()) {
+ description = metric.description();
+ }
+ }
+ }
+
+ String message = "OFF";
+ StringBuilder buffer = bufferOff;
+ if (isForwarded) {
+ buffer = bufferOn;
+ message = "ON";
+ }
+ buffer.append(m.getName()).append('=').append(message);
+ if (!description.isEmpty()) {
+ buffer.append(";description=").append(description);
+ }
+ if (!alias.isEmpty()) {
+ buffer.append(";output-name=").append(alias);
+ }
+ buffer.append(',');
+ }
+ }
+
+ return bufferOn.toString() + bufferOff.toString();
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java
new file mode 100644
index 00000000000..617cf0a1525
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java
@@ -0,0 +1,8 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+@ExportPackage
+package ai.vespa.metricsproxy.core;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java
new file mode 100644
index 00000000000..9eb1b242535
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * @author gjoranv
+ */
+public final class AggregationKey {
+
+ private Map<DimensionId, String> dimensions;
+ private Set<ConsumerId> consumers;
+
+ public AggregationKey(Map<DimensionId, String> dimensions, Set<ConsumerId> consumers) {
+ this.dimensions = dimensions;
+ this.consumers = consumers;
+ }
+
+ public Map<DimensionId, String> getDimensions() { return Collections.unmodifiableMap(dimensions); }
+
+ public Set<ConsumerId> getConsumers() { return Collections.unmodifiableSet(consumers); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AggregationKey that = (AggregationKey) o;
+ return Objects.equals(dimensions, that.dimensions) &&
+ Objects.equals(consumers, that.consumers);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dimensions, consumers);
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java
new file mode 100644
index 00000000000..62465909798
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static com.yahoo.log.LogLevel.DEBUG;
+import static java.util.stream.Collectors.toCollection;
+
+/**
+ * This class is responsible for handling metrics received from external processes.
+ *
+ * @author gjoranv
+ */
+public class ExternalMetrics {
+ private static final Logger log = Logger.getLogger(ExternalMetrics.class.getName());
+
+ public static final DimensionId ROLE_DIMENSION = toDimensionId("role");
+ public static final DimensionId STATE_DIMENSION = toDimensionId("state");
+ public static final DimensionId ORCHESTRATOR_STATE_DIMENSION = toDimensionId("orchestratorState");
+
+ static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node");
+
+ private volatile List<MetricsPacket.Builder> metrics = new ArrayList<>();
+ private final MetricsConsumers consumers;
+
+ public ExternalMetrics(MetricsConsumers consumers) {
+ this.consumers = consumers;
+ }
+
+ public List<MetricsPacket.Builder> getMetrics() {
+ return metrics;
+ }
+
+ public void setExtraMetrics(List<MetricsPacket.Builder> externalPackets) {
+ log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets.");
+ externalPackets.forEach(packet -> {
+ packet.addConsumers(consumers.getAllConsumers())
+ .service(VESPA_NODE_SERVICE_ID)
+ .retainMetrics(metricsToRetain())
+ .applyOutputNames(outputNamesById());
+ });
+ metrics = List.copyOf(externalPackets);
+ }
+
+ private Set<MetricId> metricsToRetain() {
+ return consumers.getConsumersByMetric().keySet().stream()
+ .map(configuredMetric -> toMetricId(configuredMetric.name()))
+ .collect(toCollection(LinkedHashSet::new));
+ }
+
+ /**
+ * Returns a mapping from metric id to a list of the metric's output names.
+ * Metrics that only have their id as output name are included in the output.
+ */
+ private Map<MetricId, List<String>> outputNamesById() {
+ Map<MetricId, List<String>> outputNamesById = new LinkedHashMap<>();
+ for (Consumer.Metric metric : consumers.getConsumersByMetric().keySet()) {
+ MetricId id = toMetricId(metric.name());
+ outputNamesById.computeIfAbsent(id, unused -> new ArrayList<>())
+ .add(metric.outputname());
+ }
+ return outputNamesById;
+ }
+
+ /**
+ * Extracts the node repository dimensions (role, state etc.) from the given packets.
+ * If the same dimension exists in multiple packets, this implementation gives no guarantees
+ * about which value is returned.
+ */
+ public static Map<DimensionId, String> extractConfigserverDimensions(Collection<MetricsPacket.Builder> packets) {
+ Map<DimensionId, String> dimensions = new HashMap<>();
+ for (MetricsPacket.Builder packet : packets) {
+ dimensions.putAll(packet.build().dimensions());
+ }
+ dimensions.keySet().retainAll(Set.of(ROLE_DIMENSION, STATE_DIMENSION, ORCHESTRATOR_STATE_DIMENSION));
+ return dimensions;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java
new file mode 100644
index 00000000000..41a8c3d414e
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java
@@ -0,0 +1,56 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+/**
+ * @author Jo Kristian Bergum
+ */
+public class HealthMetric {
+ private final String message;
+ private final String status;
+ private final boolean isAlive;
+
+ private HealthMetric(String status, String message, boolean isAlive) {
+ this.message = message;
+ this.status = status;
+ this.isAlive = isAlive;
+ }
+
+ public static HealthMetric get(String status, String message) {
+ if (status == null) {
+ status = "";
+ }
+ if (message == null) {
+ message = "";
+ }
+ status = status.toLowerCase();
+
+ if (status.equals("up") || status.equals("ok")) {
+ return new HealthMetric(status, message, true);
+ } else {
+ return new HealthMetric(status, message, false);
+ }
+ }
+
+ public static HealthMetric getFailed(String message) {
+ return new HealthMetric("down", message, false);
+ }
+
+ public static HealthMetric getOk(String message) {
+ return new HealthMetric("up", message, true);
+ }
+
+ public String getMessage() {
+ return this.message;
+ }
+
+ public String getStatus() {
+ return this.status;
+ }
+
+ public boolean isOk() {
+ return this.isAlive;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java
new file mode 100644
index 00000000000..59fbe301a49
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java
@@ -0,0 +1,124 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Jo Kristian Bergum
+ */
+public class Metric {
+ private final long time;
+ private final Number value;
+ private final String description;
+ private String name;
+ private Map<DimensionId, String> dimensions;
+ private Set<ConsumerId> consumers;
+
+ /**
+ * Creates a new metric instance
+ *
+ * @param name The metric name. E.g 'documents'
+ * @param value The numeric value
+ * @param time The timestamp of this metric in seconds
+ */
+ public Metric(String name, Number value, long time, Map<DimensionId, String> dimensions, String description) {
+ this.time = time;
+ this.value = value;
+ this.name = name;
+ this.dimensions = dimensions;
+ this.description = description;
+ }
+
+ public Metric(String name, Number value, long timestamp) {
+ this(name, value, timestamp, Collections.emptyMap(), "");
+ }
+
+ public Metric(String name, Number value) {
+ this(name, value, System.currentTimeMillis() / 1000);
+ }
+
+ public void setDimensions(Map<DimensionId, String> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ /**
+ * @return A map of the dimensions registered for this metric
+ */
+ public Map<DimensionId, String> getDimensions() { return dimensions; }
+
+ public void setConsumers(Set<ConsumerId> consumers) { this.consumers = consumers; }
+
+ /**
+ * @return The consumers this metric should be routed to.
+ */
+ public Set<ConsumerId> getConsumers() { return consumers; }
+
+ /**
+ * @return The number that this metric name represent
+ */
+ public Number getValue() {
+ return value;
+ }
+
+ /**
+ * Set the name of this metric
+ *
+ * @param name The name to use for this metric
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * @return The name of the metric
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return The UTC timestamp for when this metric was collected
+ */
+ public long getTimeStamp() {
+ return this.time;
+ }
+
+ @Override
+ public String toString() {
+ return "Metric{" +
+ "time=" + time +
+ ", name=" + name +
+ ", value='" + value + '\'' +
+ ", dimensions=" + dimensions +
+ '}';
+ }
+
+ @Override
+ public Metric clone() {
+ return new Metric(name, value, time, new LinkedHashMap<>(dimensions), getDescription());
+ }
+
+ /**
+ * @return the description of this metric
+ */
+ public String getDescription() {
+ return this.description;
+ }
+
+ /** Return an adjusted (rounded up) time if necessary */
+ public static long adjustTime(long timestamp, long now) {
+ if ((now == (timestamp+1)) && ((now % 60) == 0)) {
+ return now;
+ }
+ return timestamp;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java
new file mode 100644
index 00000000000..ca611368730
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java
@@ -0,0 +1,111 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Once a getter is called, the instance is frozen and no more metrics can be added.
+ *
+ * @author Unknown
+ */
+// TODO: remove timestamp, only used as temporary storage.
+// TODO: instances of this class can probably be replaced by a simple freezable map.
+public class Metrics {
+ private final List<Metric> metrics = new ArrayList<>();
+ private long timestamp;
+ private boolean isFrozen = false;
+
+ public Metrics() {
+ this(System.currentTimeMillis() / 1000L);
+ }
+
+ public Metrics(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ private void ensureNotFrozen() {
+ if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!");
+
+ }
+
+ public long getTimeStamp() {
+ return this.timestamp;
+ }
+
+ /**
+ * Update the timestamp
+ *
+ * @param timestamp IN UTC seconds resolution
+ */
+ public void setTimeStamp(long timestamp) {
+ ensureNotFrozen();
+ this.timestamp = timestamp;
+ }
+
+ public void add(Metric m) {
+ ensureNotFrozen();
+ this.timestamp = m.getTimeStamp();
+ this.metrics.add(m);
+ }
+
+ /**
+ * Get the size of the metrics covered. Note that this might also contain expired metrics
+ *
+ * @return size of metrics
+ */
+ public int size() {
+ return this.metrics.size();
+ }
+
+ /**
+ * TODO: Remove, might be multiple metrics with same name but different dimensions
+ *
+ * @param key metric name
+ * @return the metric, or null
+ */
+ public Metric getMetric(String key) {
+ isFrozen = true;
+ for (Metric m: metrics) {
+ if (m.getName().equals(key)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ public List<Metric> getMetrics() {
+ isFrozen = true;
+ return Collections.unmodifiableList(metrics);
+ }
+
+
+ /**
+ * Get a single metric based on the metric name
+ * TODO: Remove, might be multiple metrics with same name, but different
+ *
+ * @param key metric name
+ * @return The value or null if metric was not found or expired
+ */
+ public Number get(String key) {
+ isFrozen = true;
+ Metric m = getMetric(key);
+ if (m != null) {
+ return m.getValue();
+ }
+ return null;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (Metric m : metrics) {
+ sb.append(m.getName()).append(":").append(m.getValue()).append("\n");
+ }
+ return sb.toString();
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java
new file mode 100644
index 00000000000..8858e21486a
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.service.VespaService;
+
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+
+/**
+ * Format metrics as required by users of the "getMetricsById" rpc method.
+ *
+ * @author Unknown
+ */
+public class MetricsFormatter {
+ private final boolean includeServiceName;
+ private final boolean isSystemMetric;
+ private final DecimalFormat df = new DecimalFormat("0.000", new DecimalFormatSymbols(Locale.ENGLISH));
+
+ public MetricsFormatter(boolean includeServiceName, boolean isSystemMetric) {
+ this.includeServiceName = includeServiceName;
+ this.isSystemMetric = isSystemMetric;
+ }
+
+ public String format(VespaService service, String name, Number value) {
+ StringBuilder sb = new StringBuilder();
+
+ if (includeServiceName) {
+ sb.append(service.getServiceName()).append(".");
+ }
+
+ if (isSystemMetric)
+ sb.append(toSystemServiceId(service.getConfigId()));
+ else
+ sb.append(toServiceId(service.getConfigId()));
+
+ sb.append(".")
+ .append(formatMetricName(name))
+ .append("=");
+
+ if (value instanceof Double) {
+ sb.append(df.format(value.doubleValue()));
+ } else {
+ sb.append(value.toString());
+ }
+
+ return sb.toString();
+ }
+
+ private static String formatMetricName(String name) {
+ name = name.replaceAll("\"", "");
+ name = name.replaceAll("\\.", "_");
+ return name;
+ }
+
+ // E.g. container/qrserver.1 -> 'container.qrserver.1'
+ private static String toServiceId(String configId) {
+ return "'" + configId.replace("/", ".") + "'";
+ }
+
+ // E.g. container/qrserver.1 -> container.'qrserver.1'
+ private static String toSystemServiceId(String configId) {
+ String name = configId.replace("/", ".");
+ name = name.replaceFirst("\\.", ".'") + "'";
+ return name;
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java
new file mode 100644
index 00000000000..ae40f672a32
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.dimensions;
+
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+
+import java.util.Map;
+
+import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+
+/**
+ * Application-specific but node-agnostic dimensions.
+ *
+ * @author gjoranv
+ */
+public class ApplicationDimensions {
+
+ private final Map<DimensionId, String> dimensions;
+
+ public ApplicationDimensions(ApplicationDimensionsConfig config) {
+ dimensions = config.dimensions().entrySet().stream().collect(
+ toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue));
+ }
+
+ public Map<DimensionId, String> getDimensions() { return dimensions; }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java
new file mode 100644
index 00000000000..d2c1799e148
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.dimensions;
+
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+
+import java.util.Map;
+
+import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+
+/**
+ * Node-specific metric dimensions.
+ *
+ * @author gjoranv
+ */
+public class NodeDimensions {
+
+ private final Map<DimensionId, String> dimensions;
+
+ public NodeDimensions(NodeDimensionsConfig config) {
+ dimensions = config.dimensions().entrySet().stream().collect(
+ toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue));
+ }
+
+ public Map<DimensionId, String> getDimensions() { return dimensions; }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java
new file mode 100644
index 00000000000..f4e5f74313a
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java
@@ -0,0 +1,8 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+@ExportPackage
+package ai.vespa.metricsproxy.metric.dimensions;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java
new file mode 100644
index 00000000000..0d7acd5f354
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java
@@ -0,0 +1,38 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import java.util.Objects;
+
+/**
+ * @author gjoranv
+ */
+public class ConsumerId {
+ public final String id;
+ private ConsumerId(String id) { this.id = id; }
+
+ public static ConsumerId toConsumerId(String id) { return new ConsumerId(id); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ConsumerId that = (ConsumerId) o;
+ return Objects.equals(id, that.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ return "ConsumerId{" +
+ "id='" + id + '\'' +
+ '}';
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java
new file mode 100644
index 00000000000..03f4c2c01ff
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java
@@ -0,0 +1,38 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import java.util.Objects;
+
+/**
+ * @author gjoranv
+ */
+public class DimensionId {
+
+ public final String id;
+ private DimensionId(String id) { this.id = id; }
+
+ public static DimensionId toDimensionId(String id) { return new DimensionId(id); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ DimensionId that = (DimensionId) o;
+ return Objects.equals(id, that.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ return "DimensionId{" +
+ "id='" + id + '\'' +
+ '}';
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java
new file mode 100644
index 00000000000..c93735c7fca
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java
@@ -0,0 +1,39 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import java.util.Objects;
+
+/**
+ * @author gjoranv
+ */
+public class MetricId {
+
+ public final String id;
+ private MetricId(String id) { this.id = id; }
+
+ public static MetricId toMetricId(String id) { return new MetricId(id); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ MetricId metricId = (MetricId) o;
+ return Objects.equals(id, metricId.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ return "MetricId{" +
+ "id='" + id + '\'' +
+ '}';
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
new file mode 100644
index 00000000000..fa45c6251f6
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import ai.vespa.metricsproxy.metric.Metric;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static java.util.Collections.unmodifiableList;
+import static java.util.Collections.unmodifiableMap;
+import static java.util.stream.Collectors.joining;
+
+/**
+ * Represents a packet of metrics (with meta information) that belong together because they:
+ * <ul>
+ * <li>share both the same dimensions and consumers, AND</li>
+ * <li>represent the same source, e.g. a vespa service or the system hardware.</li>
+ * </ul>
+ *
+ * @author gjoranv
+ */
+public class MetricsPacket {
+ public final int statusCode;
+ public final String statusMessage;
+ public final long timestamp;
+ public final ServiceId service;
+ private final Map<MetricId, Number> metrics;
+ private final Map<DimensionId, String> dimensions;
+ private final List<ConsumerId> consumers;
+
+ private MetricsPacket(int statusCode, String statusMessage, long timestamp, ServiceId service,
+ Map<MetricId, Number> metrics, Map<DimensionId, String> dimensions, Set<ConsumerId> consumers ) {
+ this.statusCode = statusCode;
+ this.statusMessage = statusMessage;
+ this.timestamp = timestamp;
+ this.service = service;
+ this.metrics = metrics;
+ this.dimensions = dimensions;
+ this.consumers = new ArrayList<>(consumers);
+ }
+
+ public Map<MetricId, Number> metrics() {
+ return unmodifiableMap(metrics);
+ }
+
+ public Map<DimensionId, String> dimensions() {
+ return unmodifiableMap(dimensions);
+ }
+
+ public List<ConsumerId> consumers() {
+ return unmodifiableList(consumers);
+ }
+
+ @Override
+ public String toString() {
+ return "MetricsPacket{" +
+ "statusCode=" + statusCode +
+ ", statusMessage='" + statusMessage + '\'' +
+ ", timestamp=" + timestamp +
+ ", service=" + service.id +
+ ", metrics=" + idMapToString(metrics, id -> id.id) +
+ ", dimensions=" + idMapToString(dimensions, id -> id.id) +
+ ", consumers=" + consumers.stream().map(id -> id.id).collect(joining(",", "[", "]")) +
+ '}';
+ }
+
+ private static <K,V> String idMapToString(Map<K,V> map, Function<K, String> idMapper) {
+ return map.entrySet().stream()
+ .map(entry -> idMapper.apply(entry.getKey()) + "=" + entry.getValue())
+ .collect(joining(",", "{", "}"));
+ }
+
+ public static class Builder {
+ // Set sensible defaults here, and use null guard in all setters.
+ // Except for 'service' for which we require an explicit non-null value.
+ private ServiceId service;
+ private int statusCode = 0;
+ private String statusMessage = "<null>";
+ private long timestamp = 0L;
+ private Map<MetricId, Number> metrics = new LinkedHashMap<>();
+ private final Map<DimensionId, String> dimensions = new LinkedHashMap<>();
+ private final Set<ConsumerId> consumers = new LinkedHashSet<>();
+
+ public Builder(ServiceId service) {
+ Objects.requireNonNull(service, "Service cannot be null.");
+ this.service = service;
+ }
+
+ public Builder service(ServiceId service) {
+ if (service == null) throw new IllegalArgumentException("Service cannot be null.");
+ this.service = service;
+ return this;
+ }
+
+ public Builder statusCode(Integer statusCode) {
+ if (statusCode != null) this.statusCode = statusCode;
+ return this;
+ }
+
+ public Builder statusMessage(String statusMessage) {
+ if (statusMessage != null) this.statusMessage = statusMessage;
+ return this;
+ }
+
+ public Builder timestamp(Long timestamp) {
+ if (timestamp != null) this.timestamp = timestamp;
+ return this;
+ }
+
+ public Builder putMetrics(Collection<Metric> extraMetrics) {
+ if (extraMetrics != null)
+ extraMetrics.forEach(metric -> metrics.put(toMetricId(metric.getName()),
+ metric.getValue().doubleValue()));
+ return this;
+ }
+
+ public Builder putMetric(MetricId id, Number value) {
+ metrics.put(id, value);
+ return this;
+ }
+
+ public Builder retainMetrics(Set<MetricId> idsToRetain) {
+ metrics.keySet().retainAll(idsToRetain);
+ return this;
+ }
+
+ public Builder applyOutputNames(Map<MetricId, List<String>> outputNamesById) {
+ Map<MetricId, Number> newMetrics = new LinkedHashMap<>();
+ outputNamesById.forEach((id, outputNames) -> {
+ if (metrics.containsKey(id))
+ outputNames.forEach(outputName -> newMetrics.put(toMetricId(outputName), metrics.get(id)));
+ });
+ metrics = newMetrics;
+ return this;
+ }
+
+ public Builder putDimension(DimensionId id, String value) {
+ dimensions.put(id, value);
+ return this;
+ }
+
+ public Builder putDimensions(Map<DimensionId, String> extraDimensions) {
+ if (extraDimensions != null) dimensions.putAll(extraDimensions);
+ return this;
+ }
+
+ public Builder putDimensionsIfAbsent(Map<DimensionId, String> extraDimensions) {
+ if (extraDimensions != null) extraDimensions.forEach(dimensions::putIfAbsent);
+ return this;
+ }
+
+ public Builder addConsumers(Set<ConsumerId> extraConsumers) {
+ if (extraConsumers != null) consumers.addAll(extraConsumers);
+ return this;
+ }
+
+ public MetricsPacket build() {
+ return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers);
+ }
+
+ public boolean hasMetrics() {
+ return ! metrics.isEmpty();
+ }
+
+ public Instant getTimestamp() { return Instant.ofEpochSecond(timestamp); }
+
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java
new file mode 100644
index 00000000000..b61ead75b72
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java
@@ -0,0 +1,39 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import java.util.Objects;
+
+/**
+ * @author gjoranv
+ */
+public class ServiceId {
+
+ public final String id;
+ private ServiceId(String id) { this.id = id; }
+
+ public static ServiceId toServiceId(String id) { return new ServiceId(id); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ServiceId serviceId = (ServiceId) o;
+ return Objects.equals(id, serviceId.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ return "ServiceId{" +
+ "id='" + id + '\'' +
+ '}';
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java
new file mode 100644
index 00000000000..f48e5759528
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static com.yahoo.stream.CustomCollectors.toLinkedMap;
+import static java.util.Collections.emptyList;
+import static java.util.logging.Level.WARNING;
+
+/**
+ * @author gjoranv
+ */
+public class JsonUtil {
+ private static final Logger log = Logger.getLogger(JsonUtil.class.getName());
+
+ static final String YAMAS_ROUTING = "yamas";
+
+ public static MetricsPacket.Builder toMetricsPacketBuilder(YamasJsonModel jsonModel) {
+ if (jsonModel.application == null)
+ throw new IllegalArgumentException("Service id cannot be null");
+
+ return new MetricsPacket.Builder(toServiceId(jsonModel.application))
+ .statusCode(jsonModel.status_code)
+ .statusMessage(jsonModel.status_msg)
+ .timestamp(jsonModel.timestamp)
+ .putMetrics(jsonModel.getMetricsList())
+ .putDimensions(jsonModel.getDimensionsById())
+ .addConsumers(jsonModel.getYamasConsumers());
+ }
+
+ public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets) {
+ YamasArrayJsonModel yamasArray = toYamasArray(metricsPackets, false);
+
+ // Add a single status object at the end
+ yamasArray.metrics.stream().findFirst().map(YamasJsonModel::getYamasConsumers)
+ .ifPresent(consumers -> yamasArray.add(getStatusYamasModel("Data collected successfully", 0, consumers)));
+ return yamasArray;
+ }
+
+ public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets, boolean addStatus) {
+ YamasArrayJsonModel yamasArray = new YamasArrayJsonModel();
+ metricsPackets.forEach(packet -> yamasArray.add(toYamasModel(packet, addStatus)));
+ return yamasArray;
+ }
+
+ /**
+ * Converts the given json formatted string to a list of metrics packet builders.
+ * Note that this method returns an empty list if an IOException occurs,
+ * and logs a warning as a side effect.
+ */
+ public static List<MetricsPacket.Builder> toMetricsPackets(String jsonString) {
+ List<MetricsPacket.Builder> packets = new ArrayList<>();
+ try {
+ JsonParser jp = new JsonFactory().createParser(jsonString);
+ jp.setCodec(new ObjectMapper());
+ while (jp.nextToken() != null) {
+ YamasJsonModel jsonModel = jp.readValueAs(YamasJsonModel.class);
+ packets.add(toMetricsPacketBuilder(jsonModel));
+ }
+ return packets;
+ } catch (IOException e) {
+ log.log(WARNING, "Could not create metrics packet from string:\n" + jsonString, e);
+ return emptyList();
+ }
+ }
+
+ private static YamasJsonModel getStatusYamasModel(String statusMessage, int statusCode, Collection<ConsumerId> consumers) {
+ YamasJsonModel model = new YamasJsonModel();
+ model.status_code = statusCode;
+ model.status_msg = statusMessage;
+ model.application = "yms_check_vespa";
+ model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(consumers));
+ return model;
+ }
+
+ private static YamasJsonModel toYamasModel(MetricsPacket packet, boolean addStatus) {
+ YamasJsonModel model = new YamasJsonModel();
+
+ if (addStatus) {
+ model.status_code = packet.statusCode;
+ model.status_msg = packet.statusMessage;
+ }
+
+ model.application = packet.service.id;
+ model.timestamp = (packet.timestamp == 0L) ? null : packet.timestamp;
+
+ if (packet.metrics().isEmpty()) model.metrics = null;
+ else {
+ model.metrics = packet.metrics().entrySet().stream().collect(
+ toLinkedMap(id2metric -> id2metric.getKey().id,
+ id2metric -> id2metric.getValue().doubleValue()));
+ }
+
+ if (packet.dimensions().isEmpty()) model.dimensions = null;
+ else {
+ model.dimensions = packet.dimensions().entrySet().stream().collect(
+ toLinkedMap(id2dim -> id2dim.getKey().id,
+ Map.Entry::getValue));
+ }
+
+ if (packet.consumers().isEmpty()) model.routing = null;
+ else model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(packet.consumers()));
+
+ return model;
+ }
+
+ private static YamasJsonModel.YamasJsonNamespace toYamasJsonNamespaces(Collection<ConsumerId> consumers) {
+ YamasJsonModel.YamasJsonNamespace namespaces = new YamasJsonModel.YamasJsonNamespace();
+ namespaces.namespaces = consumers.stream().map(consumer -> consumer.id).collect(Collectors.toList());
+ return namespaces;
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java
new file mode 100644
index 00000000000..fdac0521256
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * Datamodel for the metricsproxy representation of multiple yamas checks.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class YamasArrayJsonModel {
+ @JsonProperty("metrics")
+ public final List<YamasJsonModel> metrics = new ArrayList<>();
+
+ public void add(List<YamasJsonModel> results) {
+ metrics.addAll(results);
+ }
+
+ public void add(YamasJsonModel result) {
+ metrics.add(result);
+ }
+
+ public void add(YamasArrayJsonModel array) {
+ metrics.addAll(array.metrics);
+ }
+
+ /**
+ * Convenience method to serialize.
+ * <p>
+ * Custom floating point serializer to avoid scientifc notation
+ *
+ * @return Serialized json
+ */
+ public String serialize() {
+ ObjectMapper mapper = new ObjectMapper();
+ SimpleModule module = new SimpleModule("DoubleSerializer",
+ new Version(1, 0, 0, "", null, null));
+ module.addSerializer(Double.class, new DoubleSerializer());
+ mapper.registerModule(module);
+
+ if (metrics.size() > 0) {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ }
+
+ return "{}"; // Backwards compatability
+ }
+
+ public class DoubleSerializer extends JsonSerializer<Double> {
+ @Override
+ public void serialize(Double value, JsonGenerator jgen,
+ SerializerProvider provider) throws IOException, JsonProcessingException {
+ DecimalFormat df = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH));
+ df.setMaximumFractionDigits(13);
+ jgen.writeNumber(df.format(value));
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java
new file mode 100644
index 00000000000..5fdbe9577be
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.stream.CustomCollectors.toLinkedMap;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+/**
+ * Datamodel for Yamas execute output
+ * <p>
+ * Used to read from original yamas checks and annotate with routing information.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ "status_code", "timestamp", "application", "metrics", "dimensions", "routing", "status_msg"})
+public class YamasJsonModel {
+ @JsonProperty("status_code")
+ public Integer status_code;
+ @JsonProperty("status_msg")
+ public String status_msg;
+ @JsonProperty("timestamp")
+ public Long timestamp;
+ @JsonProperty("application")
+ public String application;
+ @JsonProperty("metrics")
+ public Map<String, Double> metrics;
+ @JsonProperty("dimensions")
+ public Map<String, String> dimensions;
+ @JsonProperty("routing")
+ public Map<String, YamasJsonNamespace> routing;
+
+ public static class YamasJsonNamespace {
+ @JsonProperty("namespaces")
+ public List<String> namespaces;
+ }
+
+ // NOTE: do not rename to 'setMetrics', as jackson will try to use it.
+ public void resetMetrics(List<Metric> newMetrics) {
+ metrics = new LinkedHashMap<>();
+ newMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue().doubleValue()));
+ }
+
+ /**
+ * Convenience method to add targets to the routing object
+ *
+ * @param names Namespaces E.g "Vespa"
+ */
+ public void addRouting(Set<ConsumerId> names) {
+ // Setup routing structure if not already existing
+ if (routing == null) {
+ routing = new HashMap<>();
+ }
+
+ if (! routing.containsKey("yamas")) {
+ routing.put("yamas", new YamasJsonModel.YamasJsonNamespace());
+ }
+ YamasJsonModel.YamasJsonNamespace namespace = routing.get("yamas");
+
+ if (namespace.namespaces == null) {
+ namespace.namespaces = new ArrayList<>();
+ }
+
+ namespace.namespaces.addAll(names.stream().map(consumer -> consumer.id).collect(Collectors.toList()));
+ }
+
+ /**
+ * Convenience method to add dimensions
+ */
+ public void addDimensions(Map<DimensionId, String> additionalDimensions, boolean replace) {
+ additionalDimensions.forEach((k,v) -> {
+ addDimension(k.id, v, replace);
+ });
+ }
+
+ /**
+ * Convenience method to add dimensions
+ */
+ public void addDimension(String key, String value, boolean replace) {
+ if (dimensions == null) {
+ dimensions = new HashMap<>();
+ }
+ if (!dimensions.containsKey(key) || replace) {
+ dimensions.put(key, value);
+ }
+ }
+
+ List<Metric> getMetricsList() {
+ if (metrics == null) return emptyList();
+
+ return metrics.keySet().stream()
+ .map(name -> new Metric(name, metrics.get(name)))
+ .collect(Collectors.toList());
+ }
+
+ Map<DimensionId, String> getDimensionsById() {
+ if (dimensions == null) return emptyMap();
+
+ return dimensions.keySet().stream().collect(toLinkedMap(DimensionId::toDimensionId,
+ name -> dimensions.get(name)));
+ }
+
+ Set<ConsumerId> getYamasConsumers() {
+ if (routing == null || routing.get("yamas") == null) return emptySet();
+
+ return routing.get("yamas").namespaces.stream()
+ .map(ConsumerId::toConsumerId)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java
new file mode 100644
index 00000000000..c72f2484f8c
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java
@@ -0,0 +1,8 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+@ExportPackage
+package ai.vespa.metricsproxy.metric;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java
new file mode 100644
index 00000000000..e7feab9926d
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.rpc;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.jrt.Acceptor;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.Method;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Transport;
+
+import java.util.logging.Logger;
+
+import static com.yahoo.log.LogLevel.DEBUG;
+import static java.util.logging.Level.INFO;
+
+/**
+ * Contains the connector for the rpc server, to prevent it from going down after component reconfiguration.
+ * This will only be recreated if the rpc port changes, which should never happen under normal circumstances.
+ *
+ * @author gjoranv
+ */
+public class RpcConnector extends AbstractComponent {
+ private static final Logger log = Logger.getLogger(RpcConnector.class.getName());
+
+ private final Supervisor supervisor = new Supervisor(new Transport());
+ private final Acceptor acceptor;
+
+ public RpcConnector(RpcConnectorConfig config) {
+ Spec spec = new Spec(config.port());
+ try {
+ acceptor = supervisor.listen(spec);
+ log.log(DEBUG, "Listening on " + spec.host() + ":" + spec.port());
+ } catch (ListenFailedException e) {
+ stop();
+ log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port());
+ throw new RuntimeException("Could not listen at " + spec, e);
+ }
+ }
+
+ /**
+ * Adds a method. If a method with the same name already exists, it will be replaced.
+ * @param method The method to add.
+ */
+ public void addMethod(Method method) {
+ supervisor.addMethod(method);
+ }
+
+ public void stop() {
+ acceptor.shutdown().join();
+ supervisor.transport().shutdown().join();
+ }
+
+ @Override
+ public void deconstruct() {
+ stop();
+ super.deconstruct();
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java
new file mode 100644
index 00000000000..e0e0e7a3f87
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.rpc;
+
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.yahoo.jrt.ErrorCode;
+import com.yahoo.jrt.Method;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Transport;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets;
+import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toYamasArray;
+import static com.yahoo.collections.CollectionUtil.mkString;
+import static com.yahoo.log.LogLevel.DEBUG;
+import static java.util.logging.Level.INFO;
+import static java.util.logging.Level.WARNING;
+
+/**
+ * Rpc server for the metrics proxy.
+ *
+ * When a new object is created after reconfiguration, it will claim ownership of the methods
+ * in the given {@link RpcConnector}. This is ok because at the time this component is created,
+ * all components it depends on are already created.
+ *
+ * @author gjoranv
+ */
+public class RpcServer {
+
+ private static final Logger log = Logger.getLogger(RpcServer.class.getName());
+
+ private static int LOG_SPENT_TIME_LIMIT = 10 * 1000; // ms. same as default client RPC timeout used in rpc_invoke
+
+ private final VespaServices vespaServices;
+ private final MetricsManager metricsManager;
+
+ public RpcServer(RpcConnector connector, VespaServices vespaServices, MetricsManager metricsManager) {
+ this.vespaServices = vespaServices;
+ this.metricsManager = metricsManager;
+ addMethods(connector);
+ log.log(DEBUG, "RPC server created");
+ }
+
+ private void addMethods(RpcConnector connector) {
+ // Add/replace this method first to increase likelihood of getting extra metrics and global dimensions
+ connector.addMethod(
+ new Method("setExtraMetrics", "s", "", this::setExtraMetrics)
+ .methodDesc("Set extra metrics that will be added to output from getMetricsForYamas.")
+ .paramDesc(0, "metricsJson", "The metrics in json format"));
+
+ connector.addMethod(
+ new Method("getMetricsById", "s", "s", this::getMetricsById)
+ .methodDesc("Get Vespa metrics for the service with the given Id")
+ .paramDesc(0, "id", "The id of the service")
+ .returnDesc(0, "ret", "Vespa metrics"));
+
+ connector.addMethod(
+ new Method("getServices", "", "s", this::getServices)
+ .methodDesc("Get Vespa services monitored by this metrics proxy")
+ .returnDesc(0, "ret", "Vespa metrics"));
+
+ connector.addMethod(
+ new Method("getMetricsForYamas", "s", "s", this::getMetricsForYamas)
+ .methodDesc("Get JSON formatted Vespa metrics for a given service name or 'all'")
+ .paramDesc(0, "service", "The vespa service name, or 'all'")
+ .returnDesc(0, "ret", "Vespa metrics"));
+
+ connector.addMethod(
+ new Method("getHealthMetricsForYamas", "s", "s", this::getHealthMetricsForYamas)
+ .methodDesc("Get JSON formatted Health check for a given service name or 'all'")
+ .paramDesc(0, "service", "The vespa service name")
+ .returnDesc(0, "ret", "Vespa metrics"));
+
+ connector.addMethod(
+ new Method("getAllMetricNamesForService", "ss", "s", this::getAllMetricNamesForService)
+ .methodDesc("Get metric names known for service ")
+ .paramDesc(0, "service", "The vespa service name'")
+ .paramDesc(1, "consumer", "The consumer'")
+ .returnDesc(0, "ret", "Metric names, one metric name per line"));
+ }
+
+ void getAllMetricNamesForService(Request req) {
+ String service = req.parameters().get(0).asString();
+ ConsumerId consumer = toConsumerId(req.parameters().get(1).asString());
+ withExceptionHandling(req, () -> {
+ String metricNames = metricsManager.getMetricNamesForServiceAndConsumer(service, consumer);
+ req.returnValues().add(new StringValue(metricNames));
+ });
+ }
+
+ void getMetricsById(Request req) {
+ String id = req.parameters().get(0).asString();
+ withExceptionHandling(req, () -> {
+ String metricsString = metricsManager.getMetricsByConfigId(id);
+ req.returnValues().add(new StringValue(metricsString));
+ });
+ }
+
+
+ void getServices(Request req) {
+ withExceptionHandling(req, () -> {
+ String servicesString = metricsManager.getAllVespaServices();
+ req.returnValues().add(new StringValue(servicesString));
+ });
+ }
+
+ void getMetricsForYamas(Request req) {
+ Instant startTime = Instant.now();
+ req.detach();
+ String service = req.parameters().get(0).asString();
+ log.log(DEBUG, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service);
+ List<VespaService> services = vespaServices.getMonitoringServices(service);
+ log.log(DEBUG, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]"));
+ if (services.isEmpty()) setNoServiceError(req, service);
+ else withExceptionHandling(req, () -> {
+ List<MetricsPacket> packets = metricsManager.getMetrics(services, startTime);
+ log.log(DEBUG,() -> "Returning metrics packets:\n" + mkString(packets, "\n"));
+ req.returnValues().add(new StringValue(toYamasArray(packets).serialize()));
+ });
+ req.returnRequest();
+ }
+
+ void getHealthMetricsForYamas(Request req) {
+ req.detach();
+ String service = req.parameters().get(0).asString();
+ List<VespaService> services = vespaServices.getMonitoringServices(service);
+ if (services.isEmpty()) setNoServiceError(req, service);
+ else withExceptionHandling(req, () -> {
+ List<MetricsPacket> packets = metricsManager.getHealthMetrics(services);
+ req.returnValues().add(new StringValue(toYamasArray(packets, true).serialize()));
+ });
+ req.returnRequest();
+ }
+
+ void setExtraMetrics(Request req) {
+ String metricsJson = req.parameters().get(0).asString();
+ log.log(DEBUG, "setExtraMetrics called with argument: " + metricsJson);
+ withExceptionHandling(req, () -> metricsManager.setExtraMetrics(toMetricsPackets(metricsJson)));
+ }
+
+ private static void withExceptionHandling(Request req, ThrowingRunnable runnable) {
+ try {
+ TimeTracker timeTracker = new TimeTracker(req);
+ runnable.run();
+ timeTracker.logSpentTime();
+ } catch (Exception e) {
+ log.log(WARNING, "Got exception when running RPC command " + req.methodName(), e);
+ setMethodFailedError(req, e);
+ } catch (Error e) {
+ log.log(WARNING, "Got error when running RPC command " + req.methodName(), e);
+ setMethodFailedError(req, e);
+ } catch (Throwable t) {
+ log.log(WARNING, "Got throwable (non-error, non-exception) when running RPC command " + req.methodName(), t);
+ setMethodFailedError(req, t);
+ }
+ }
+
+ private static void setMethodFailedError(Request req, Throwable t) {
+ String msg = "Request failed due to internal error: " + t.getClass().getName() + ": " + t.getMessage();
+ req.setError(ErrorCode.METHOD_FAILED, msg);
+ req.returnValues().add(new StringValue(""));
+ }
+
+ private static void setNoServiceError(Request req, String serviceName) {
+ String msg = "No service with name '" + serviceName + "'";
+ req.setError(ErrorCode.BAD_REQUEST, msg);
+ req.returnValues().add(new StringValue(""));
+ }
+
+
+ private static class TimeTracker {
+ private final long startTime = System.currentTimeMillis();
+ private final Request request;
+
+ private TimeTracker(Request request) {
+ this.request = request;
+ }
+
+ long spentTime() {
+ return System.currentTimeMillis() - startTime;
+ }
+
+ private void logSpentTime() {
+ Level logLevel = DEBUG;
+ if (spentTime() > LOG_SPENT_TIME_LIMIT) {
+ logLevel = INFO;
+ }
+ if (log.isLoggable(logLevel)) {
+ log.log(logLevel, "RPC request '" + request.methodName() + "' with parameters '" +
+ request.parameters() + "' took " + spentTime() + " ms");
+ }
+ }
+ }
+
+ private interface ThrowingRunnable {
+ void run() throws Exception;
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java
new file mode 100644
index 00000000000..875b6190763
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import com.google.inject.Inject;
+import com.yahoo.log.LogLevel;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Connects to the config sentinel and gets information like pid for the services on the node
+ */
+public class ConfigSentinelClient {
+ private final static Logger log = Logger.getLogger(ConfigSentinelClient.class.getName());
+
+ private final CmdClient client;
+
+ @Inject
+ public ConfigSentinelClient() {
+ this.client = new CmdClient();
+ }
+
+ /**
+ * Update all services reading from config sentinel
+ *
+ * @param services The list of services
+ */
+ synchronized void updateServiceStatuses(List<VespaService> services) {
+ try {
+ setStatus(services);
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e);
+ }
+ }
+
+
+ /**
+ * Update status
+ *
+ * @param s The service to update the status for
+ */
+ public synchronized void ping(VespaService s) {
+ List<VespaService> services = new ArrayList<>();
+ services.add(s);
+ log.log(LogLevel.DEBUG, "Ping for service " + s);
+ try {
+ setStatus(services);
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e);
+ }
+ }
+
+ /**
+ * Update the status (pid check etc)
+ *
+ * @param services list of services
+ * @throws Exception if something went wrong
+ */
+ protected synchronized void setStatus(List<VespaService> services) throws Exception {
+ InputStream in;
+ PrintStream out;
+ client.connect();
+
+ in = client.getInputStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ String line;
+ List<VespaService> updatedServices = new ArrayList<>();
+ while ((line = reader.readLine()) != null) {
+ if (line.equals("")) {
+ break;
+ }
+
+ VespaService s = parseServiceString(line, services);
+ if (s != null) {
+ updatedServices.add(s);
+ }
+ }
+
+ //Check if there are services that were not found in output
+ //from the sentinel
+ for (VespaService s : services) {
+ if ((!s.getServiceName().equals("configserver")) && !updatedServices.contains(s)) {
+ log.log(LogLevel.DEBUG,"Service " + s + " is no longer found with sentinel - setting alive = false");
+ s.setAlive(false);
+ }
+ }
+
+ //Close streams
+ reader.close();
+ client.disconnect();
+ }
+
+ protected static VespaService parseServiceString(String line, List<VespaService> services) {
+ String[] parts = line.split(" ");
+ if (parts.length < 3)
+ return null;
+
+ String name = parts[0];
+ int pid = -1;
+ String state = null;
+ VespaService service = null;
+
+ for (VespaService s : services) {
+ if (s.getInstanceName().compareToIgnoreCase(name) == 0) {
+ service = s;
+ break;
+ }
+ }
+
+ //Could not find this service
+ //nothing wrong with that as the check is invoked per line from sentinel
+ if (service == null) {
+ return service;
+ }
+
+ for (int i = 1; i < parts.length; i++) {
+ String keyValue[] = parts[i].split("=");
+
+ String key = keyValue[0];
+ String value = keyValue[1];
+
+ if (key.equals("state")) {
+ state = value;
+ } else if (key.equals("pid")) {
+ pid = Integer.parseInt(value);
+ }
+ }
+
+ if (state != null) {
+ service.setState(state);
+ if (pid >= 0 && "RUNNING".equals(state)) {
+ service.setAlive(true);
+ service.setPid(pid);
+ } else {
+ service.setAlive(false);
+
+ }
+ } else {
+ service.setAlive(false);
+ }
+ return service;
+ }
+
+ static class CmdClient {
+ Process proc;
+ // NOTE: hostname/port not used yet
+ void connect() {
+ String[] args = new String[]{"vespa-sentinel-cmd", "list"};
+ try {
+ proc = Runtime.getRuntime().exec(args);
+ } catch (Exception e) {
+ log.log(LogLevel.WARNING, "could not run vespa-sentinel-cmd: "+e);
+ proc = null;
+ }
+ }
+ void disconnect() {
+ if (proc.isAlive()) {
+ proc.destroy();
+ }
+ proc = null;
+ }
+ InputStream getInputStream() {
+ return (proc != null)
+ ? proc.getInputStream()
+ : new java.io.ByteArrayInputStream(new byte[0]);
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java
new file mode 100644
index 00000000000..312e7d5c0c1
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+class CpuJiffies {
+ private int cpuId;
+ private long jiffies;
+
+ CpuJiffies(String line) {
+ parseLine(line);
+ }
+
+ private void parseLine(String line) {
+ String elems[];
+ String cpuId;
+ long jiffies;
+
+ elems = line.split("\\s+");
+ cpuId = elems[0].substring(3);
+ if (cpuId.length() == 0) {
+ this.cpuId = -1;
+ } else {
+ this.cpuId = Integer.parseInt(cpuId);
+ }
+
+ jiffies = 0;
+ for (int i = 1; i < elems.length; i++) {
+ jiffies += Long.parseLong(elems[i].replaceAll("[\\n\\r]+", ""));
+ }
+
+ this.jiffies = jiffies;
+ }
+
+ public int getCpuId() {
+ return cpuId;
+ }
+
+ public long getTotalJiffies() {
+ return jiffies;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java
new file mode 100644
index 00000000000..f87171a42dc
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.HealthMetric;
+
+/**
+ * Dummy class used for getting health status for a vespa service that has no HTTP service
+ * for getting health status
+ *
+ * @author hmusum
+ */
+public class DummyHealthMetricFetcher extends RemoteHealthMetricFetcher {
+
+ /**
+ * @param service The service to fetch metrics from
+ */
+ DummyHealthMetricFetcher(VespaService service) {
+ super(service, 0);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ public HealthMetric getHealth(int fetchCount) {
+ if (service.isAlive()) {
+ return HealthMetric.getOk("Service is running - pid check only");
+ } else {
+ return HealthMetric.getFailed("Service is not running - pid check only");
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java
new file mode 100644
index 00000000000..f21d125e279
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.Metrics;
+
+/**
+ * Dummy class used for getting health status for a vespa service that has no HTTP service
+ * for getting metrics
+ *
+ * @author hmusum
+ */
+public class DummyMetricsFetcher extends RemoteMetricsFetcher {
+
+ /**
+ * @param service The service to fetch metrics from
+ */
+ DummyMetricsFetcher(VespaService service) {
+ super(service, 0);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ public Metrics getMetrics(int fetchCount) {
+ return new Metrics();
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java
new file mode 100644
index 00000000000..9094ef22c20
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java
@@ -0,0 +1,94 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.util.http.VespaHttpClientBuilder;
+import com.yahoo.log.LogLevel;
+import com.yahoo.yolean.Exceptions;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.CloseableHttpClient;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.logging.Logger;
+
+/**
+ * HTTP client to get metrics or health data from a service
+ *
+ * @author hmusum
+ * @author bjorncs
+ */
+public abstract class HttpMetricFetcher {
+ private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName());
+ public final static String STATE_PATH = "/state/v1/";
+ final static String METRICS_PATH = STATE_PATH + "metrics";
+ final static String HEALTH_PATH = STATE_PATH + "health";
+ // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high.
+ public static int CONNECTION_TIMEOUT = 5000;
+ private final static int SOCKET_TIMEOUT = 60000;
+ private final URI url;
+ protected final VespaService service;
+ private static final CloseableHttpClient httpClient = createHttpClient();
+
+
+ /**
+ * @param service The service to fetch metrics from
+ * @param port The port to use
+ */
+ HttpMetricFetcher(VespaService service, int port, String path) {
+ this.service = service;
+
+ String u = "http://localhost:" + port + path;
+ this.url = URI.create(u);
+ log.log(LogLevel.DEBUG, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT);
+ }
+
+ String getJson() throws IOException {
+ log.log(LogLevel.DEBUG, "Connecting to url " + url + " for service '" + service + "'");
+ return httpClient.execute(new HttpGet(url), new BasicResponseHandler());
+ }
+
+ public String toString() {
+ return this.getClass().getSimpleName() + " using " + url;
+ }
+
+ String errMsgNoResponse(IOException e) {
+ return "Unable to get response from service '" + service + "': " +
+ Exceptions.toMessageString(e);
+ }
+
+ void handleException(Exception e, String data, int timesFetched) {
+ logMessage("Unable to parse json '" + data + "' for service '" + service + "': " +
+ Exceptions.toMessageString(e), timesFetched);
+ }
+
+ private void logMessage(String message, int timesFetched) {
+ if (service.isAlive() && timesFetched > 5) {
+ log.log(LogLevel.INFO, message);
+ } else {
+ log.log(LogLevel.DEBUG, message);
+ }
+ }
+
+ void logMessageNoResponse(String message, int timesFetched) {
+ if (timesFetched > 5) {
+ log.log(LogLevel.WARNING, message);
+ } else {
+ log.log(LogLevel.INFO, message);
+ }
+ }
+
+ private static CloseableHttpClient createHttpClient() {
+ return VespaHttpClientBuilder.create()
+ .setUserAgent("metrics-proxy-http-client")
+ .setDefaultRequestConfig(RequestConfig.custom()
+ .setConnectTimeout(CONNECTION_TIMEOUT)
+ .setSocketTimeout(SOCKET_TIMEOUT)
+ .build())
+ .build();
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java
new file mode 100644
index 00000000000..503f582a827
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java
@@ -0,0 +1,77 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.HealthMetric;
+import com.yahoo.log.LogLevel;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+/**
+ * Fetch health status for a given vespa service
+ *
+ * @author Jo Kristian Bergum
+ */
+public class RemoteHealthMetricFetcher extends HttpMetricFetcher {
+
+ private final static Logger log = Logger.getLogger(RemoteHealthMetricFetcher.class.getPackage().getName());
+
+ /**
+ * @param service The service to fetch metrics from
+ * @param port The port to use
+ */
+ public RemoteHealthMetricFetcher(VespaService service, int port) {
+ super(service, port, HEALTH_PATH);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ public HealthMetric getHealth(int fetchCount) {
+ String data = "{}";
+ try {
+ data = getJson();
+ } catch (IOException e) {
+ logMessageNoResponse(errMsgNoResponse(e), fetchCount);
+ }
+ return createHealthMetrics(data, fetchCount);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ HealthMetric createHealthMetrics(String data, int fetchCount) {
+ HealthMetric healthMetric = HealthMetric.getFailed("Failed fetching status page for service");
+ try {
+ healthMetric = parse(data);
+ } catch (Exception e) {
+ handleException(e, data, fetchCount);
+ }
+ return healthMetric;
+ }
+
+ private HealthMetric parse(String data) {
+ if (data == null || data.isEmpty()) {
+ return HealthMetric.getFailed("Empty response from status page");
+ }
+ try {
+ JSONObject o = new JSONObject(data);
+ JSONObject status = o.getJSONObject("status");
+ String code = status.getString("code");
+ String message = "";
+ if (status.has("message")) {
+ message = status.getString("message");
+ }
+ return HealthMetric.get(code, message);
+
+ } catch (JSONException e) {
+ log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data);
+ return HealthMetric.getFailed("Not able to parse json from status page");
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java
new file mode 100644
index 00000000000..a606ec7d8cd
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java
@@ -0,0 +1,129 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+
+/**
+ * Fetch metrics for a given vespa service
+ *
+ * @author Jo Kristian Bergum
+ */
+public class RemoteMetricsFetcher extends HttpMetricFetcher {
+ /**
+ * @param service The service to fetch metrics from
+ * @param port The port to use
+ */
+ RemoteMetricsFetcher(VespaService service, int port) {
+ super(service, port, METRICS_PATH);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ public Metrics getMetrics(int fetchCount) {
+ String data = "{}";
+ try {
+ data = getJson();
+ } catch (IOException e) {
+ logMessageNoResponse(errMsgNoResponse(e), fetchCount);
+ }
+
+ return createMetrics(data, fetchCount);
+ }
+
+ /**
+ * Connect to remote service over http and fetch metrics
+ */
+ public Metrics createMetrics(String data, int fetchCount) {
+ Metrics remoteMetrics = new Metrics();
+ try {
+ remoteMetrics = parse(data);
+ } catch (Exception e) {
+ handleException(e, data, fetchCount);
+ }
+
+ return remoteMetrics;
+ }
+
+ Metrics parse(String data) throws JSONException {
+ JSONObject o = new JSONObject(data);
+ if (!(o.has("metrics"))) {
+ return new Metrics(); //empty
+ }
+
+ JSONObject metrics = o.getJSONObject("metrics");
+ JSONArray values;
+ long timestamp;
+
+ try {
+ JSONObject snapshot = metrics.getJSONObject("snapshot");
+ timestamp = (long) snapshot.getDouble("to");
+ values = metrics.getJSONArray("values");
+ } catch (JSONException e) {
+ // snapshot might not have been produced. Do not throw exception into log
+ return new Metrics();
+ }
+ long now = System.currentTimeMillis() / 1000;
+ timestamp = Metric.adjustTime(timestamp, now);
+ Metrics m = new Metrics(timestamp);
+
+ Map<DimensionId, String> noDims = Collections.emptyMap();
+ Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>();
+ for (int i = 0; i < values.length(); i++) {
+ JSONObject metric = values.getJSONObject(i);
+ String name = metric.getString("name");
+ String description = "";
+
+ if (metric.has("description")) {
+ description = metric.getString("description");
+ }
+
+ Map<DimensionId, String> dim = noDims;
+ if (metric.has("dimensions")) {
+ JSONObject dimensions = metric.getJSONObject("dimensions");
+ StringBuilder sb = new StringBuilder();
+ for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) {
+ String k = (String) it.next();
+ String v = dimensions.getString(k);
+ sb.append(toDimensionId(k)).append(v);
+ }
+ if ( ! uniqueDimensions.containsKey(sb.toString())) {
+ dim = new HashMap<>();
+ for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) {
+ String k = (String) it.next();
+ String v = dimensions.getString(k);
+ dim.put(toDimensionId(k), v);
+ }
+ uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim));
+ }
+ dim = uniqueDimensions.get(sb.toString());
+ }
+
+ JSONObject aggregates = metric.getJSONObject("values");
+ for (Iterator<?> it = aggregates.keys(); it.hasNext(); ) {
+ String aggregator = (String) it.next();
+ Number value = (Number) aggregates.get(aggregator);
+ StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator);
+ m.add(new Metric(metricName.toString(), value, timestamp, dim, description));
+ }
+ }
+
+ return m;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java
new file mode 100644
index 00000000000..810eb5fb908
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java
@@ -0,0 +1,14 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import java.util.List;
+
+/**
+ * @author Unknown
+ */
+public interface ServiceListener {
+ void setServices(List<VespaService> services);
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
new file mode 100644
index 00000000000..9f6614668a5
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import com.yahoo.log.LogLevel;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Logger;
+
+/**
+ * Class to get data from the system and update the services at given intervals.
+ * TODO: rewrite to use ScheduledExecutorService or just call poll() directly.
+ *
+ * @author Eirik Nygaard
+ */
+public class SystemPoller implements ServiceListener {
+ final private static Logger log = Logger.getLogger(SystemPoller.class.getPackage().getName());
+
+ private final int pollingIntervalSecs;
+ private volatile List<VespaService> services;
+
+ private final int memoryTypeVirtual = 0;
+ private final int memoryTypeResident = 1;
+ private final Map<VespaService, Long> lastCpuJiffiesMetrics = new ConcurrentHashMap<>();
+ private final Timer systemPollTimer;
+
+ private long lastTotalCpuJiffies = -1;
+
+ public SystemPoller(List<VespaService> services, int pollingIntervalSecs) {
+ this.services = services;
+ this.pollingIntervalSecs = pollingIntervalSecs;
+ systemPollTimer = new Timer("systemPollTimer", true);
+ }
+
+ @Override
+ public void setServices(List<VespaService> services) {
+ log.log(LogLevel.DEBUG, "Setting services in SystemPoller to: " + services);
+ this.services = services;
+ }
+
+ void stop() {
+ systemPollTimer.cancel();
+ }
+
+ /**
+ * Return memory usage for a given process, both resident and virtual is
+ * returned.
+ *
+ * @param service The instance to get memory usage for
+ * @return array[0] = memoryResident, array[1] = memoryVirtual (kB units)
+ */
+ long[] getMemoryUsage(VespaService service) {
+ long size[] = new long[2];
+ BufferedReader br;
+ int pid = service.getPid();
+
+ size[0] = 0;
+ size[1] = 0;
+ try {
+ br = new BufferedReader(new FileReader("/proc/" + pid + "/smaps"));
+ } catch (FileNotFoundException ex) {
+ markDead(service);
+ return size;
+ }
+ String line;
+ try {
+ while ((line = br.readLine()) != null) {
+ String[] elems = line.split("\\s+");
+ /* Memory size is given in kB - convert to bytes by multiply with 1024*/
+ if (line.startsWith("Rss:")) {
+ size[memoryTypeResident] += Long.parseLong(elems[1]) * 1024;
+ } else if (line.startsWith("Size:")) {
+ size[memoryTypeVirtual] += Long.parseLong(elems[1]) * 1024;
+ }
+ }
+
+ br.close();
+ } catch (IOException ex) {
+ log.log(LogLevel.DEBUG, "Unable to read line from smaps file", ex);
+ return size;
+ }
+
+ return size;
+ }
+
+ /**
+ * Mark a service as dead.
+ *
+ * @param service The service to mark as dead.
+ */
+ private static void markDead(VespaService service) {
+ service.setAlive(false);
+ }
+
+ /**
+ * Poll services for system metrics
+ */
+ void poll() {
+ long startTime = System.currentTimeMillis();
+ boolean someAlive = false;
+
+ /* Don't do any work if there are no known services */
+ if (services.isEmpty()) {
+ schedule();
+ return;
+ }
+
+ log.log(LogLevel.DEBUG, "Monitoring system metrics for " + services.size() + " services");
+
+ long sysJiffies = getNormalizedSystemJiffies();
+ for (VespaService s : services) {
+
+
+ if(s.isAlive()) {
+ someAlive = true;
+ }
+
+ Metrics metrics = new Metrics();
+ log.log(LogLevel.DEBUG, "Current size of system metrics for service " + s + " is " + metrics.size());
+
+ long[] size = getMemoryUsage(s);
+ log.log(LogLevel.DEBUG, "Updating memory metric for service " + s);
+
+ metrics.add(new Metric("memory_virt", size[memoryTypeVirtual], startTime / 1000));
+ metrics.add(new Metric("memory_rss", size[memoryTypeResident], startTime / 1000));
+
+ long procJiffies = getPidJiffies(s);
+ if (lastTotalCpuJiffies >= 0 && lastCpuJiffiesMetrics.containsKey(s)) {
+ long last = lastCpuJiffiesMetrics.get(s);
+ long diff = procJiffies - last;
+
+ if (diff >= 0) {
+ metrics.add(new Metric("cpu", 100 * ((double) diff) / (sysJiffies - lastTotalCpuJiffies), startTime / 1000));
+ }
+ }
+ lastCpuJiffiesMetrics.put(s, procJiffies);
+ s.setSystemMetrics(metrics);
+ }
+
+ lastTotalCpuJiffies = sysJiffies;
+
+ // If none of the services were alive, reschedule in a short time
+ if (!someAlive) {
+ reschedule(System.currentTimeMillis() - startTime);
+ } else {
+ schedule();
+ }
+ }
+
+ long getPidJiffies(VespaService service) {
+ BufferedReader in;
+ String line;
+ String[] elems;
+ int pid = service.getPid();
+
+ try {
+ in = new BufferedReader(new FileReader("/proc/" + pid + "/stat"));
+ } catch (FileNotFoundException ex) {
+ log.log(LogLevel.DEBUG, "Unable to find pid in proc directory " + pid);
+ service.setAlive(false);
+ return 0;
+ }
+
+ try {
+ line = in.readLine();
+ in.close();
+ } catch (IOException ex) {
+ log.log(LogLevel.DEBUG, "Unable to read line from process stat file", ex);
+ return 0;
+ }
+
+ elems = line.split(" ");
+
+ /* Add user mode and kernel mode jiffies for the given process */
+ return Long.parseLong(elems[13]) + Long.parseLong(elems[14]);
+ }
+
+ long getNormalizedSystemJiffies() {
+ BufferedReader in;
+ String line;
+ ArrayList<CpuJiffies> jiffies = new ArrayList<>();
+ CpuJiffies total = null;
+
+ try {
+ in = new BufferedReader(new FileReader("/proc/stat"));
+ } catch (FileNotFoundException ex) {
+ log.log(LogLevel.ERROR, "Unable to open stat file", ex);
+ return 0;
+ }
+ try {
+ while ((line = in.readLine()) != null) {
+ if (line.startsWith("cpu ")) {
+ total = new CpuJiffies(line);
+ } else if (line.startsWith("cpu")) {
+ jiffies.add(new CpuJiffies(line));
+ }
+ }
+
+ in.close();
+ } catch (IOException ex) {
+ log.log(LogLevel.ERROR, "Unable to read line from stat file", ex);
+ return 0;
+ }
+
+ /* Normalize so that a process that uses an entire CPU core will get 100% util */
+ if (total != null) {
+ return total.getTotalJiffies() / jiffies.size();
+ } else {
+ return 0;
+ }
+ }
+
+ private void schedule(long time) {
+ try {
+ systemPollTimer.schedule(new PollTask(this), time);
+ } catch(IllegalStateException e){
+ log.info("Tried to schedule task, but timer was already shut down.");
+ }
+ }
+
+ public void schedule() {
+ schedule(pollingIntervalSecs * 1000);
+ }
+
+ private void reschedule(long skew) {
+ long sleep = (pollingIntervalSecs * 1000) - skew;
+
+ // Don't sleep less than 1 min
+ sleep = Math.max(60 * 1000, sleep);
+ schedule(sleep);
+ }
+
+
+ private static class PollTask extends TimerTask {
+ private final SystemPoller poller;
+
+ PollTask(SystemPoller poller) {
+ this.poller = poller;
+ }
+
+ @Override
+ public void run() {
+ poller.poll();
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java
new file mode 100644
index 00000000000..dea7b2b4809
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.core.MonitoringConfig;
+import com.yahoo.container.di.componentgraph.Provider;
+
+/**
+ * @author gjoranv
+ */
+public class SystemPollerProvider implements Provider<SystemPoller> {
+
+ private final SystemPoller poller;
+
+ /**
+ * @param services The list of VespaService instances to monitor for System metrics
+ * @param monitoringConfig The interval in seconds between each polling.
+ */
+ public SystemPollerProvider (VespaServices services, MonitoringConfig monitoringConfig) {
+ poller = new SystemPoller(services.getVespaServices(), 60 * monitoringConfig.intervalMinutes());
+ poller.poll();
+ }
+
+ public void deconstruct() {
+ poller.stop();
+ }
+
+ public SystemPoller get() {
+ return poller;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
new file mode 100644
index 00000000000..d3f674176b3
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.HealthMetric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+/**
+ * Represents a Vespa service
+ *
+ * @author jobergum
+ */
+public class VespaService implements Comparable<VespaService> {
+
+ private static final Map<DimensionId, String> EMPTY_DIMENSIONS = Collections.emptyMap();
+ private static final String DEFAULT_MONITORING_PREFIX = "vespa";
+ public static final String SEPARATOR = ".";
+
+ private final String instanceName;
+ private final String configId;
+ private final String serviceName;
+ private final String monitoringPrefix;
+ private final Map<DimensionId, String> dimensions;
+
+
+ private volatile int pid = -1;
+ private volatile String state = "UNKNOWN";
+
+ // Used to keep the last polled system metrics for service
+ private Metrics systemMetrics;
+
+ private final int statePort;
+
+ private final RemoteHealthMetricFetcher remoteHealthMetricFetcher;
+ private final RemoteMetricsFetcher remoteMetricsFetcher;
+
+ private boolean isAlive;
+
+ // Used to keep track of log level when health or metrics requests fail
+ private AtomicInteger metricsFetchCount = new AtomicInteger(0);
+ private AtomicInteger healthFetchCount = new AtomicInteger(0);
+
+
+ public static VespaService create(String name, String id, int statePort) {
+ return create(name,id, statePort, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS);
+ }
+
+ public static VespaService create(String name, String id, int statePort, String monitoringName, Map<DimensionId, String> dimensions) {
+ String serviceName = name.replaceAll("\\d*$", "");
+ return new VespaService(serviceName, name, id, statePort, monitoringName, dimensions);
+ }
+
+ VespaService(String serviceName, String configId) {
+ this(serviceName, serviceName, configId);
+ }
+
+ VespaService(String serviceName, String instanceName, String configId) {
+ this(serviceName, instanceName, configId, -1, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS);
+ }
+
+ private VespaService(String serviceName, String instanceName, String configId,
+ int statePort, String monitoringPrefix,
+ Map<DimensionId, String> dimensions) {
+ this.serviceName = serviceName;
+ this.instanceName = instanceName;
+ this.monitoringPrefix = monitoringPrefix;
+ this.configId = configId;
+ this.statePort = statePort;
+ this.dimensions = dimensions;
+ this.systemMetrics = new Metrics();
+ this.isAlive = false;
+ this.remoteMetricsFetcher = (this.statePort> 0) ? new RemoteMetricsFetcher(this, this.statePort) : new DummyMetricsFetcher(this);
+ this.remoteHealthMetricFetcher = (this.statePort > 0) ? new RemoteHealthMetricFetcher(this, this.statePort) : new DummyHealthMetricFetcher(this);
+ }
+
+ /**
+ * The name used for this service in the monitoring system:
+ * monitoring-system-name.serviceName
+ */
+ public String getMonitoringName() {
+ return monitoringPrefix + SEPARATOR + serviceName;
+ }
+
+ @Override
+ public int compareTo(VespaService other) {
+ return this.getInstanceName().compareTo(other.getInstanceName());
+ }
+
+ /**
+ * Get the service name/type. E.g 'searchnode', but not 'searchnode2'
+ *
+ * @return the service name
+ */
+ public String getServiceName() {
+ return this.serviceName;
+ }
+
+ /**
+ * Get the instance name. E.g searchnode2
+ *
+ * @return the instance service name
+ */
+ public String getInstanceName() {
+ return this.instanceName;
+ }
+
+ public Map<DimensionId, String> getDimensions() {
+ return dimensions;
+ }
+
+ /**
+ * @return The health of this service
+ */
+ public HealthMetric getHealth() {
+ HealthMetric healthMetric = remoteHealthMetricFetcher.getHealth(healthFetchCount.get());
+ healthFetchCount.getAndIncrement();
+ return healthMetric;
+ }
+
+ /**
+ * Gets the system metrics for this service
+ *
+ * @return System metrics
+ */
+ public synchronized Metrics getSystemMetrics() {
+ return this.systemMetrics;
+ }
+
+ /**
+ * Get the Metrics registered for this service. Metrics are fetched over HTTP
+ * if a metric http port has been defined, otherwise from log file
+ *
+ * @return the non-system metrics
+ */
+ public Metrics getMetrics() {
+ Metrics remoteMetrics = remoteMetricsFetcher.getMetrics(metricsFetchCount.get());
+ metricsFetchCount.getAndIncrement();
+ return remoteMetrics;
+ }
+
+ /**
+ * Gets the config id of this service
+ *
+ * @return the config id
+ */
+ public String getConfigId() {
+ return configId;
+ }
+
+ /**
+ * The current pid of this service
+ *
+ * @return The pid
+ */
+ public int getPid() {
+ return this.pid;
+ }
+
+ /**
+ * update the pid of this service
+ *
+ * @param pid The pid that this service runs as
+ */
+ public void setPid(int pid) {
+ this.pid = pid;
+ }
+
+ /**
+ * Get the string representation of the state of this service
+ *
+ * @return string representing the state of this service - obtained from config-sentinel
+ */
+ public String getState() {
+ return state;
+ }
+
+ /**
+ * Update the state of this service
+ *
+ * @param state the new state
+ */
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ /**
+ * Check if this pid/service is running
+ *
+ * @return true if the service is alive (e.g the pid is running)
+ */
+ public boolean isAlive() {
+ return (isAlive && (pid >= 0));
+ }
+
+ @Override
+ public String toString() {
+ return instanceName + ":" + pid + ":" + state + ":" + configId;
+ }
+
+ public void setAlive(boolean alive) {
+ this.isAlive = alive;
+ }
+
+ public synchronized void setSystemMetrics(Metrics systemMetrics) {
+ this.systemMetrics = systemMetrics;
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java
new file mode 100644
index 00000000000..2668c158ed6
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.core.MonitoringConfig;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.service.VespaServicesConfig.Service;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static com.yahoo.log.LogLevel.DEBUG;
+
+/**
+ * Creates representations for the Vespa services running on the node,
+ * and provides methods for updating and getting them.
+ *
+ * @author gjoranv
+ */
+public class VespaServices {
+ private static final Logger log = Logger.getLogger(VespaServices.class.getName());
+
+ public static final String ALL_SERVICES = "all";
+
+ private final ConfigSentinelClient sentinel;
+ private final List<VespaService> services;
+
+ @Inject
+ public VespaServices(VespaServicesConfig config, MonitoringConfig monitoringConfig, ConfigSentinelClient sentinel) {
+ this.services = createServices(config, monitoringConfig.systemName());
+ this.sentinel = sentinel;
+ }
+
+ @VisibleForTesting
+ public VespaServices(List<VespaService> services) {
+ this.services = services;
+ sentinel = null;
+ }
+
+ private List<VespaService> createServices(VespaServicesConfig servicesConfig, String monitoringSystemName) {
+ List<VespaService> services = new ArrayList<>();
+ for (Service s : servicesConfig.service()) {
+ log.log(DEBUG, "Re-configuring service " + s.name());
+ VespaService vespaService = VespaService.create(s.name(), s.configId(), s.healthport(), monitoringSystemName,
+ createServiceDimensions(s));
+ services.add(vespaService);
+ }
+ log.log(DEBUG, "Created new services: " + services.size());
+ updateServices(services);
+ return services;
+ }
+
+ /**
+ * Sets 'alive=false' for services that are no longer running.
+ * Note that the status is updated in-place for the given services.
+ */
+ public void updateServices(List<VespaService> services) {
+ if (sentinel != null) {
+ log.log(DEBUG, "Updating services ");
+ sentinel.updateServiceStatuses(services);
+ }
+ }
+
+ /**
+ * Get all known vespa services
+ *
+ * @return A list of VespaService objects
+ */
+ public List<VespaService> getVespaServices() {
+ return Collections.unmodifiableList(services);
+ }
+
+ /**
+ * @param id The configid
+ * @return A list with size 1 as there should only be one service with the given configid
+ */
+ public List<VespaService> getInstancesById(String id) {
+ List<VespaService> myServices = new ArrayList<>();
+ for (VespaService s : services) {
+ if (s.getConfigId().equals(id)) {
+ myServices.add(s);
+ }
+ }
+
+ return myServices;
+ }
+
+ /**
+ * Get services matching pattern for the name used in the monitoring system.
+ *
+ * @param service name in monitoring system + service name, without index, e.g: vespa.container
+ * @return A list of VespaServices
+ */
+ public List<VespaService> getMonitoringServices(String service) {
+ if (service.equalsIgnoreCase(ALL_SERVICES))
+ return services;
+
+ List<VespaService> myServices = new ArrayList<>();
+ for (VespaService s : services) {
+ log.log(DEBUG, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName());
+ if (s.getMonitoringName().equalsIgnoreCase(service)) {
+ myServices.add(s);
+ }
+ }
+
+ return myServices;
+ }
+
+ private static Map<DimensionId, String> createServiceDimensions(Service service) {
+ return service.dimension().stream().collect(
+ toUnmodifiableLinkedMap(dim -> toDimensionId(dim.key()), Service.Dimension::value));
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java
new file mode 100644
index 00000000000..b478cdf8b5b
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java
@@ -0,0 +1,8 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+@ExportPackage
+package ai.vespa.metricsproxy.service;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def
new file mode 100644
index 00000000000..41b384193e5
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def
@@ -0,0 +1,5 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.metric.dimensions
+
+# Dimensions based on application properties
+dimensions{} string
diff --git a/metrics-proxy/src/main/resources/configdefinitions/consumers.def b/metrics-proxy/src/main/resources/configdefinitions/consumers.def
new file mode 100644
index 00000000000..a1828b1320c
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/consumers.def
@@ -0,0 +1,10 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.core
+
+# Consumers with metric definitions
+consumer[].name string default=""
+consumer[].metric[].name string
+consumer[].metric[].description string default=""
+consumer[].metric[].outputname string
+consumer[].metric[].dimension[].key string
+consumer[].metric[].dimension[].value string
diff --git a/metrics-proxy/src/main/resources/configdefinitions/monitoring.def b/metrics-proxy/src/main/resources/configdefinitions/monitoring.def
new file mode 100644
index 00000000000..6fe24c86394
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/monitoring.def
@@ -0,0 +1,8 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.core
+
+# The rate at which metrics are passed to the monitoring system. Currently (Apr 2019) only used by SystemPoller.
+intervalMinutes int default=5
+
+# The name used for this application in the monitoring system
+systemName string default=vespa
diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def
new file mode 100644
index 00000000000..917cd165a9f
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def
@@ -0,0 +1,5 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.metric.dimensions
+
+# Dimensions based on node properties
+dimensions{} string
diff --git a/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def b/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def
new file mode 100644
index 00000000000..7a8450780de
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def
@@ -0,0 +1,4 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.rpc
+
+port int
diff --git a/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def b/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def
new file mode 100644
index 00000000000..6fdacfffd42
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def
@@ -0,0 +1,10 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.service
+
+# Services with service id and the http port for the metrics page
+service[].configId string
+service[].name string default=""
+service[].port int default=-1
+service[].healthport int default=-1
+service[].dimension[].key string
+service[].dimension[].value string
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java
new file mode 100644
index 00000000000..5997f85a46f
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.stream.Collectors;
+
+/**
+ * @author gjoranv
+ */
+public class TestUtil {
+
+ public static String getContents(String filename) {
+ InputStream in = TestUtil.class.getClassLoader().getResourceAsStream(filename);
+ if (in == null) {
+ throw new RuntimeException("File not found: " + filename);
+ }
+ return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n"));
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java
new file mode 100644
index 00000000000..81634323269
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.core;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer;
+import ai.vespa.metricsproxy.metric.ExternalMetrics;
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.service.DummyService;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+import static ai.vespa.metricsproxy.core.MetricsManager.VESPA_VERSION;
+import static ai.vespa.metricsproxy.core.VespaMetrics.METRIC_TYPE_DIMENSION_ID;
+import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID;
+import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class MetricsManagerTest {
+
+ private MetricsManager metricsManager;
+
+ private static final String SERVICE_0_ID = "dummy/id/0";
+ private static final String SERVICE_1_ID = "dummy/id/1";
+
+ private static final List<VespaService> testServices = ImmutableList.of(
+ new DummyService(0, SERVICE_0_ID),
+ new DummyService(1, SERVICE_1_ID));
+
+ private static final String WHITELISTED_METRIC_ID = "whitelisted";
+
+ @Before
+ public void setupMetricsManager() {
+ metricsManager = getMetricsManager();
+ }
+
+ @Test
+ public void each_service_gets_separate_metrics_packets() {
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertThat(packets.size(), is(2));
+
+ assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is("dummy0"));
+ assertThat(packets.get(0).metrics().get(toMetricId("c.test")), is(1.0));
+ assertThat(packets.get(0).metrics().get(toMetricId("val")), is(1.05));
+
+ assertThat(packets.get(1).dimensions().get(toDimensionId("instance")), is("dummy1"));
+ assertThat(packets.get(1).metrics().get(toMetricId("c.test")), is(6.0));
+ assertThat(packets.get(1).metrics().get(toMetricId("val")), is(2.35));
+ }
+
+ @Test
+ public void verify_expected_output_from_getMetricsById() {
+ String dummy0Metrics = metricsManager.getMetricsByConfigId("dummy/id/0");
+ assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050"));
+ assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1"));
+
+ String dummy1Metrics = metricsManager.getMetricsByConfigId("dummy/id/1");
+ assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350"));
+ assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6"));
+ }
+
+ @Test
+ public void getServices_returns_service_types() {
+ assertThat(metricsManager.getAllVespaServices(), is("dummy"));
+ }
+
+ @Test
+ public void global_dimensions_are_added_but_do_not_override_metric_dimensions() {
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertEquals(2, packets.size());
+ assertGlobalDimensions(packets.get(0).dimensions());
+ assertGlobalDimensions(packets.get(1).dimensions());
+ }
+
+ private void assertGlobalDimensions(Map<DimensionId, String> dimensions) {
+ assertTrue(dimensions.containsKey(VESPA_VERSION));
+ assertEquals("value", dimensions.get(toDimensionId("global")));
+ assertEquals("metric-dim", dimensions.get(toDimensionId("dim0")));
+ }
+
+
+ @Test
+ public void system_metrics_are_added() {
+ VespaService service0 = testServices.get(0);
+ Metrics oldSystemMetrics = service0.getSystemMetrics();
+
+ service0.getSystemMetrics().add(new Metric("cpu", 1));
+
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertEquals(3, packets.size());
+
+ MetricsPacket systemPacket = packets.get(0); // system metrics are added before other metrics
+ assertThat(systemPacket.metrics().get(toMetricId("cpu")), is(1.0));
+ assertThat(systemPacket.dimensions().get(toDimensionId("metrictype")), is("system"));
+
+ service0.setSystemMetrics(oldSystemMetrics);
+ }
+
+ @Test
+ public void extra_metrics_packets_containing_whitelisted_metrics_are_added() {
+ metricsManager.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0)))));
+
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertThat(packets.size(), is(3));
+ }
+
+ @Test
+ public void extra_metrics_packets_without_whitelisted_metrics_are_not_added() {
+ metricsManager.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(new Metric("not-whitelisted", 0)))));
+
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertThat(packets.size(), is(2));
+ }
+
+ @Test
+ public void extra_dimensions_are_added_to_metrics_packets_that_do_not_have_those_dimensions() {
+ metricsManager.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0)))
+ .putDimension(ROLE_DIMENSION, "role from extraMetrics")));
+
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ for (MetricsPacket packet : packets) {
+ assertThat(packet.dimensions().get(ROLE_DIMENSION), is("role from extraMetrics"));
+ }
+ }
+
+ @Test
+ public void extra_dimensions_do_not_overwrite_existing_dimension_values() {
+ metricsManager.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0)))
+ .putDimension(METRIC_TYPE_DIMENSION_ID, "from extraMetrics")));
+
+ List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH);
+ assertThat(packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard"));
+ assertThat(packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard"));
+ assertThat(packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("from extraMetrics"));
+ }
+
+ @Test
+ public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_younger_than_start_time() {
+ Instant START_TIME = Instant.ofEpochSecond(0);
+ Instant METRIC_TIME = Instant.ofEpochSecond(59);
+ assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME));
+ }
+
+ @Test
+ public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_older_than_start_time() {
+ Instant START_TIME = Instant.ofEpochSecond(59);
+ Instant METRIC_TIME = Instant.ofEpochSecond(0);
+ assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME));
+ }
+
+ @Test
+ public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_younger_than_start_time() {
+ Instant START_TIME = Instant.ofEpochSecond(0);
+ Instant METRIC_TIME = Instant.ofEpochSecond(60);
+ assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME));
+ }
+
+ @Test
+ public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_older_than_start_time() {
+ Instant START_TIME = Instant.ofEpochSecond(60);
+ Instant METRIC_TIME = Instant.ofEpochSecond(0);
+ assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME));
+ }
+
+ private Instant getAdjustedTimestamp(Instant startTime, Instant metricTime) {
+ MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId("foo"))
+ .timestamp(metricTime.getEpochSecond());
+ return MetricsManager.adjustTimestamp(builder, startTime).getTimestamp();
+ }
+
+ private MetricsManager getMetricsManager() {
+ VespaServices vespaServices = new VespaServices(testServices);
+ MetricsConsumers consumers = getMetricsConsumers();
+ VespaMetrics metrics = new VespaMetrics(consumers, vespaServices);
+
+ return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers),
+ getApplicationDimensions(),getNodeDimensions());
+ }
+
+ private static MetricsConsumers getMetricsConsumers() {
+ Consumer.Metric.Dimension.Builder metricDimension = new Consumer.Metric.Dimension.Builder()
+ .key("dim0").value("metric-dim");
+
+ return new MetricsConsumers(new ConsumersConfig.Builder()
+ .consumer(new Consumer.Builder()
+ .name(VESPA_CONSUMER_ID.id)
+ .metric(new Consumer.Metric.Builder()
+ .name(WHITELISTED_METRIC_ID)
+ .outputname(WHITELISTED_METRIC_ID))
+ .metric(new Consumer.Metric.Builder()
+ .name(DummyService.METRIC_1)
+ .outputname(DummyService.METRIC_1)
+ .dimension(metricDimension))
+ .metric(new Consumer.Metric.Builder()
+ .name(DummyService.METRIC_2)
+ .outputname(DummyService.METRIC_2)
+ .dimension(metricDimension)))
+ .build());
+ }
+
+ private ApplicationDimensions getApplicationDimensions() {
+ return new ApplicationDimensions(new ApplicationDimensionsConfig.Builder()
+ .dimensions("global", "value").build());
+ }
+
+ private NodeDimensions getNodeDimensions() {
+ return new NodeDimensions(new NodeDimensionsConfig.Builder()
+ .dimensions("dim0", "should not override metric dim").build());
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java
new file mode 100644
index 00000000000..11c271d46e4
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import java.util.List;
+
+import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author gjoranv
+ */
+public class ExternalMetricsTest {
+ private static final ConsumerId CUSTOM_CONSUMER_1 = toConsumerId("consumer-1");
+ private static final ConsumerId CUSTOM_CONSUMER_2 = toConsumerId("consumer-2");
+
+ @Test
+ public void extra_metrics_are_added() {
+ MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build());
+ ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers);
+
+ externalMetrics.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))));
+
+ List<MetricsPacket.Builder> packets = externalMetrics.getMetrics();
+ assertEquals(1, packets.size());
+ }
+
+ @Test
+ public void service_id_is_set_to_vespa_node_id() {
+ MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build());
+ ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers);
+ externalMetrics.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("replace_with_vespa_node_id"))));
+
+ List<MetricsPacket.Builder> packets = externalMetrics.getMetrics();
+ assertEquals(1, packets.size());
+ assertEquals(VESPA_NODE_SERVICE_ID, packets.get(0).build().service);
+ }
+
+ @Test
+ public void custom_consumers_are_added() {
+ ConsumersConfig consumersConfig = new ConsumersConfig.Builder()
+ .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_1.id))
+ .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_2.id))
+ .build();
+ MetricsConsumers consumers = new MetricsConsumers(consumersConfig);
+ ExternalMetrics externalMetrics = new ExternalMetrics(consumers);
+
+ externalMetrics.setExtraMetrics(ImmutableList.of(
+ new MetricsPacket.Builder(toServiceId("foo"))));
+
+ List<MetricsPacket.Builder> packets = externalMetrics.getMetrics();
+ assertEquals(1, packets.size());
+
+ List<ConsumerId> consumerIds = packets.get(0).build().consumers();
+ assertEquals(2, consumerIds.size());
+ assertEquals(CUSTOM_CONSUMER_1, consumerIds.get(0));
+ assertEquals(CUSTOM_CONSUMER_2, consumerIds.get(1));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java
new file mode 100644
index 00000000000..b9e6377c27b
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric;
+
+import ai.vespa.metricsproxy.service.DummyService;
+import ai.vespa.metricsproxy.service.VespaService;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknowm
+ */
+public class MetricsTest {
+
+ @Test
+ public void testIterator() {
+ Metrics m = new Metrics();
+ long now = System.currentTimeMillis() / 1000;
+ m.add(new Metric("a", 1, now));
+ m.add(new Metric("b", 2.5, now));
+
+ //should expire after 0 seconds
+ m.add(new Metric("c", 2, now));
+
+ Map<String, Number> map = new HashMap<>();
+
+ for (Metric metric: m.getMetrics()) {
+ String k = metric.getName();
+
+ assertThat(map.containsKey(k), is(false));
+ map.put(k, metric.getValue());
+ }
+
+ assertThat(map.get("a").intValue(), is(1));
+ assertThat(map.get("b").doubleValue(), is(2.5));
+ }
+
+ @Test
+ public void testBasicMetric() {
+ Metrics m = new Metrics();
+ m.add(new Metric("count", 1, System.currentTimeMillis() / 1000));
+ assertThat(m.get("count").intValue(), is(1));
+ }
+
+ @Test
+ public void testHealthMetric() {
+ HealthMetric m = HealthMetric.get(null, null);
+ assertThat(m.isOk(), is(false));
+ m = HealthMetric.get("up", "test message");
+ assertThat(m.isOk(), is(true));
+ assertThat(m.getMessage(), is("test message"));
+ m = HealthMetric.get("ok", "test message");
+ assertThat(m.isOk(), is(true));
+ assertThat(m.getMessage(), is("test message"));
+
+ m = HealthMetric.get("bad", "test message");
+ assertThat(m.isOk(), is(false));
+ assertThat(m.getStatus(), is("bad"));
+ }
+
+ @Test
+ public void testMetricFormatter() {
+ MetricsFormatter formatter = new MetricsFormatter(false, false);
+ VespaService service = new DummyService(0, "config.id");
+ String data = formatter.format(service, "key", 1);
+ assertThat(data, is("'config.id'.key=1"));
+
+ formatter = new MetricsFormatter(true, false);
+ data = formatter.format(service, "key", 1);
+ assertThat(data, is("dummy.'config.id'.key=1"));
+
+
+ formatter = new MetricsFormatter(true, true);
+ data = formatter.format(service, "key", 1);
+ assertThat(data, is("dummy.config.'id'.key=1"));
+
+ formatter = new MetricsFormatter(false, true);
+ data = formatter.format(service, "key", 1);
+ assertThat(data, is("config.'id'.key=1"));
+ }
+
+ @Test
+ public void testTimeAdjustment() {
+ assertThat(Metric.adjustTime(0L, 0L), is(0L));
+ assertThat(Metric.adjustTime(59L, 59L), is(59L));
+ assertThat(Metric.adjustTime(60L, 60L), is(60L));
+ assertThat(Metric.adjustTime(59L, 60L), is(60L));
+ assertThat(Metric.adjustTime(60L, 59L), is(60L));
+ assertThat(Metric.adjustTime(59L, 61L), is(59L));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
new file mode 100644
index 00000000000..d522a56a9ac
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model;
+
+import ai.vespa.metricsproxy.metric.Metric;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author gjoranv
+ */
+public class MetricsPacketTest {
+
+ @Test
+ public void service_cannot_be_null() {
+ try {
+ MetricsPacket packet = new MetricsPacket.Builder(null)
+ .statusCode(0)
+ .statusMessage("")
+ .timestamp(0L)
+ .build();
+ fail("Expected exception due to null service.");
+ } catch (Exception e) {
+ assertEquals("Service cannot be null.", e.getMessage());
+ }
+ }
+
+ @Test
+ public void consumers_are_always_distinct() {
+ ConsumerId DUPLICATE_CONSUMER = toConsumerId("duplicateConsumer");
+
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .statusCode(0)
+ .statusMessage("")
+ .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
+ .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
+ .build();
+ assertEquals(1, packet.consumers().size());
+ }
+
+ @Test
+ public void builder_can_retain_subset_of_metrics() {
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(
+ new Metric("remove", 1),
+ new Metric("keep", 2)))
+ .retainMetrics(ImmutableSet.of(toMetricId("keep"), toMetricId("non-existent")))
+ .build();
+
+ assertFalse("should not contain 'remove'", packet.metrics().containsKey(toMetricId("remove")));
+ assertTrue("should contain 'keep'", packet.metrics().containsKey(toMetricId("keep")));
+ assertFalse("should not contain 'non-existent'", packet.metrics().containsKey(toMetricId("non-existent")));
+ }
+
+ @Test
+ public void builder_applies_output_names() {
+ String ONE = "one";
+ String TWO = "two";
+ String THREE = "three";
+ String NON_EXISTENT = "non-existent";
+ MetricId ONE_ID = toMetricId(ONE);
+ MetricId TWO_ID = toMetricId(TWO);
+ MetricId THREE_ID = toMetricId(THREE);
+ MetricId NON_EXISTENT_ID = toMetricId(NON_EXISTENT);
+
+ Map<MetricId, List<String>> outputNamesById = ImmutableMap.of(
+ toMetricId(ONE), ImmutableList.of(ONE),
+ toMetricId(TWO), ImmutableList.of(TWO, "dos"),
+ toMetricId(THREE), ImmutableList.of("3"),
+ toMetricId(NON_EXISTENT), ImmutableList.of(NON_EXISTENT));
+
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .putMetrics(ImmutableList.of(
+ new Metric(ONE, 1),
+ new Metric(TWO, 2),
+ new Metric(THREE, 3)))
+ .applyOutputNames(outputNamesById)
+ .build();
+
+ // Only original name
+ assertTrue(packet.metrics().containsKey(ONE_ID));
+
+ // Both names
+ assertTrue(packet.metrics().containsKey(TWO_ID));
+ assertTrue(packet.metrics().containsKey(toMetricId("dos")));
+
+ // Only new name
+ assertFalse(packet.metrics().containsKey(THREE_ID));
+ assertTrue(packet.metrics().containsKey(toMetricId("3")));
+
+ // Non-existent metric not added
+ assertFalse(packet.metrics().containsKey(NON_EXISTENT_ID));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java
new file mode 100644
index 00000000000..28912293fdb
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import java.util.List;
+
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets;
+import static java.util.Collections.singleton;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class JsonUtilTest {
+ @Test
+ public void json_model_gets_null_status_by_default() {
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .build();
+ YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0);
+ assertNull(jsonModel.status_code);
+ assertNull(jsonModel.status_msg);
+ }
+
+ @Test
+ public void status_is_included_in_json_model_when_explicitly_asked_for() {
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .build();
+ YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet), true).metrics.get(0);
+ assertNotNull(jsonModel.status_code);
+ assertNotNull(jsonModel.status_msg);
+ }
+
+ @Test
+ public void timestamp_0_in_packet_is_translated_to_null_in_json_model() {
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .timestamp(0L)
+ .build();
+ YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0);
+ assertNull(jsonModel.timestamp);
+ }
+
+ @Test
+ public void empty_consumers_is_translated_to_null_routing_in_json_model() {
+ MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
+ .build();
+ YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0);
+ assertNull(jsonModel.routing);
+ }
+
+ @Test
+ public void empty_json_string_yields_empty_packet_list() {
+ List<MetricsPacket.Builder> builders = toMetricsPackets("");
+ assertTrue(builders.isEmpty());
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java
new file mode 100644
index 00000000000..e91ff32e3b4
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tests for YamasJsonModel and YamasArrayJsonModel
+ *
+ * @author smorgrav
+ * @author gjoranv
+ */
+public class YamasJsonModelTest {
+
+ private static final String EXPECTED_JSON = "{\"metrics\":[{\"status_code\":0,\"timestamp\":1400047900,\"application\":\"vespa.searchnode\",\"metrics\":{\"cpu\":55.5555555555555,\"memory_virt\":22222222222,\"memory_rss\":5555555555},\"dimensions\":{\"applicationName\":\"app\",\"tenantName\":\"tenant\",\"metrictype\":\"system\",\"instance\":\"searchnode\",\"applicationInstance\":\"default\",\"clustername\":\"cluster\"},\"routing\":{\"yamas\":{\"namespaces\":[\"Vespa\"]}},\"status_msg\":\"Data collected successfully\"}]}";
+
+ @Test
+ public void array_definition_creates_correct_json() throws IOException {
+ YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json");
+
+ YamasArrayJsonModel yamasData = new YamasArrayJsonModel();
+ yamasData.add(jsonModel);
+
+ assertEquals(EXPECTED_JSON, yamasData.serialize());
+ }
+
+ @Test
+ public void deserialize_serialize_roundtrip() throws IOException {
+ YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json");
+
+ // Do some sanity checking
+ assertEquals("vespa.searchnode", jsonModel.application);
+ assertEquals("Vespa", jsonModel.routing.get("yamas").namespaces.get(0));
+ assertEquals(5.555555555E9, jsonModel.metrics.get("memory_rss"), 0.1d); //Not using custom double renderer
+
+ // Serialize and verify
+ YamasArrayJsonModel yamasArray = new YamasArrayJsonModel();
+ yamasArray.add(jsonModel);
+ String string = yamasArray.serialize();
+ assertEquals(EXPECTED_JSON, string);
+ }
+
+ @Test
+ public void deserialize_serialize_roundtrip_with_metrics_packet() throws IOException {
+ YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json");
+ MetricsPacket metricsPacket = JsonUtil.toMetricsPacketBuilder(jsonModel).build();
+
+ // Do some sanity checking
+ assertEquals(toServiceId("vespa.searchnode"), metricsPacket.service);
+ assertEquals(toConsumerId("Vespa"), metricsPacket.consumers().get(0));
+ assertEquals(5.555555555E9, metricsPacket.metrics().get(toMetricId("memory_rss")).doubleValue(), 0.1d); //Not using custom double rendrer
+
+ // Serialize and verify
+ YamasArrayJsonModel yamasArray = JsonUtil.toYamasArray(Collections.singleton(metricsPacket), true);
+ String string = yamasArray.serialize();
+ assertEquals(EXPECTED_JSON, string);
+ }
+
+ @Test
+ public void missing_routing_object_makes_it_null() throws IOException {
+ // Read file that was taken from production (real -life example that is)
+ String filename = getClass().getClassLoader().getResource("yamas-array-no-routing.json").getFile();
+ BufferedReader reader = Files.newBufferedReader(Paths.get(filename));
+ ObjectMapper mapper = new ObjectMapper();
+ YamasJsonModel jsonModel = mapper.readValue(reader, YamasJsonModel.class);
+
+ // Do some sanity checking
+ assertNull(jsonModel.routing);
+ }
+
+ private YamasJsonModel getYamasJsonModel(String testFile) throws IOException {
+ String filename = getClass().getClassLoader().getResource(testFile).getFile();
+ BufferedReader reader = Files.newBufferedReader(Paths.get(filename));
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(reader, YamasJsonModel.class);
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java
new file mode 100644
index 00000000000..6bbf4ae1ef5
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.rpc;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer;
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.core.MonitoringConfig;
+import ai.vespa.metricsproxy.core.VespaMetrics;
+import ai.vespa.metricsproxy.metric.ExternalMetrics;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+import ai.vespa.metricsproxy.service.HttpMetricFetcher;
+import ai.vespa.metricsproxy.service.MockHttpServer;
+import ai.vespa.metricsproxy.service.VespaServices;
+import ai.vespa.metricsproxy.service.VespaServicesConfig;
+import ai.vespa.metricsproxy.service.VespaServicesConfig.Service;
+
+import java.io.IOException;
+
+import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID;
+import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static ai.vespa.metricsproxy.service.HttpMetricFetcher.STATE_PATH;
+
+
+/**
+ * Setup and shutdown of config and servers for integration-style unit tests.
+ *
+ * @author hmusum
+ * @author gjoranv
+ */
+public class IntegrationTester implements AutoCloseable {
+
+ static final String MONITORING_SYSTEM = "test-system";
+ static final ConsumerId CUSTOM_CONSUMER_ID = toConsumerId("custom-consumer");
+ static final String SERVICE_1_CONFIG_ID = "container/qrserver.0";
+ static final String SERVICE_2_CONFIG_ID = "storage/cluster.storage/storage/0";
+
+ private final int httpPort;
+ private final int rpcPort;
+ private final RpcConnector connector;
+ private final MockHttpServer mockHttpServer;
+ private final VespaServices vespaServices;
+
+ static {
+ HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests
+ }
+
+ IntegrationTester(int httpPort, int rpcPort) {
+ if (httpPort == 0 || rpcPort == 0) {
+ throw new IllegalArgumentException("http port and rpc port must be defined");
+ }
+ this.httpPort = httpPort;
+ this.rpcPort = rpcPort;
+ try {
+ mockHttpServer = new MockHttpServer(httpPort, null, STATE_PATH);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to start web server on port:" + httpPort);
+ }
+
+ vespaServices = new VespaServices(servicesConfig(), monitoringConfig(), null);
+ MetricsConsumers consumers = new MetricsConsumers(consumersConfig());
+ VespaMetrics vespaMetrics = new VespaMetrics(consumers, vespaServices);
+ ExternalMetrics externalMetrics = new ExternalMetrics(consumers);
+ ApplicationDimensions appDimensions = new ApplicationDimensions(applicationDimensionsConfig());
+ NodeDimensions nodeDimensions = new NodeDimensions(nodeDimensionsConfig());
+
+ connector = new RpcConnector(rpcConnectorConfig());
+ RpcServer server = new RpcServer(connector, vespaServices, new MetricsManager(vespaServices, vespaMetrics, externalMetrics, appDimensions, nodeDimensions));
+ }
+
+ MockHttpServer httpServer() {
+ return mockHttpServer;
+ }
+
+ VespaServices vespaServices() { return vespaServices; }
+
+ @Override
+ public void close() {
+ mockHttpServer.close();
+ this.connector.stop();
+ }
+
+ private RpcConnectorConfig rpcConnectorConfig() {
+ return new RpcConnectorConfig.Builder()
+ .port(rpcPort)
+ .build();
+ }
+
+ private ConsumersConfig consumersConfig() {
+ return new ConsumersConfig.Builder()
+ .consumer(createConsumer(VESPA_CONSUMER_ID, "foo.count", "foo_count"))
+ .consumer(createConsumer(CUSTOM_CONSUMER_ID, "foo.count", "foo.count"))
+ .build();
+ }
+
+ private static Consumer.Builder createConsumer(ConsumerId consumerId, String metricName, String outputName) {
+ return new Consumer.Builder()
+ .name(consumerId.id)
+ .metric(new Consumer.Metric.Builder()
+ .dimension(new Consumer.Metric.Dimension.Builder().key("somekey").value("somevalue"))
+ .name(metricName)
+ .outputname(outputName));
+ }
+
+ private VespaServicesConfig servicesConfig() {
+ return new VespaServicesConfig.Builder()
+ .service(createService(toServiceId("qrserver"), SERVICE_1_CONFIG_ID, httpPort))
+ .service(createService(toServiceId("storagenode"), SERVICE_2_CONFIG_ID, httpPort))
+ .build();
+ }
+
+ private static Service.Builder createService(ServiceId serviceId, String configId, int port) {
+ return new Service.Builder()
+ .name(serviceId.id)
+ .configId(configId)
+ .port(port)
+ .healthport(port)
+ .dimension(new Service.Dimension.Builder().key("serviceDim").value("serviceDimValue"));
+ }
+
+ private MonitoringConfig monitoringConfig() {
+ return new MonitoringConfig.Builder()
+ .systemName(MONITORING_SYSTEM)
+ .build();
+ }
+
+ private ApplicationDimensionsConfig applicationDimensionsConfig() {
+ return new ApplicationDimensionsConfig.Builder().build();
+ }
+
+ private NodeDimensionsConfig nodeDimensionsConfig() {
+ return new NodeDimensionsConfig.Builder().build();
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java
new file mode 100644
index 00000000000..20fb69e410e
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.rpc;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.metric.HealthMetric;
+import ai.vespa.metricsproxy.service.MockHttpServer;
+import ai.vespa.metricsproxy.service.VespaService;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import org.junit.Test;
+
+import java.util.List;
+
+import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author jobergum
+ * @author gjoranv
+ */
+public class RpcHealthMetricsTest {
+
+ private static final String HEALTH_OK_RESPONSE =
+ TestUtil.getContents("health-check.response.json");
+ private static final String HEALTH_FAILED_RESPONSE =
+ TestUtil.getContents("health-check-failed.response.json");
+ private static final String WANTED_RPC_RESPONSE =
+ TestUtil.getContents("rpc-json-output-check.json").trim();
+
+ // see factory/doc/port-ranges.txt
+ private static final int httpPort = 18635;
+ private static final int rpcPort = 18636;
+
+ @Test
+ public void expected_response_is_returned() {
+ try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) {
+
+ MockHttpServer mockHttpServer = tester.httpServer();
+ mockHttpServer.setResponse(HEALTH_OK_RESPONSE);
+ List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID);
+
+ assertThat(services.size(), is(1));
+ VespaService qrserver = services.get(0);
+ HealthMetric h = qrserver.getHealth();
+ assertNotNull("Health metric should never be null", h);
+ assertThat("Status failed, reason = " + h.getMessage(), h.isOk(), is(true));
+ assertThat(h.getMessage(), is("WORKING"));
+
+ mockHttpServer.setResponse(HEALTH_FAILED_RESPONSE);
+ h = qrserver.getHealth();
+ assertNotNull("Health metric should never be null", h);
+ assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false));
+ assertThat(h.getMessage(), is("SOMETHING FAILED"));
+
+ String jsonRPCMessage = getHealthMetrics(qrserver.getMonitoringName());
+ assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE));
+ }
+ }
+
+ @Test
+ public void non_existent_service_name_returns_an_error_message() {
+ try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) {
+ String jsonRPCMessage = getHealthMetrics("non-existing service");
+ assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'"));
+ }
+ }
+
+ private String getHealthMetrics(String service) {
+ Supervisor supervisor = new Supervisor(new Transport());
+ Target target = supervisor.connect(new Spec("localhost", rpcPort));
+ Request req = new Request("getHealthMetricsForYamas");
+ req.parameters().add(new StringValue(service));
+ String returnValue;
+
+ target.invokeSync(req, 20.0);
+ if (req.checkReturnTypes("s")) {
+ returnValue = req.returnValues().get(0).asString();
+ } else {
+ System.out.println("RpcServer invocation failed " + req.errorCode() + ": " + req.errorMessage());
+ returnValue = req.errorCode() + ": " + req.errorMessage();
+ }
+ target.close();
+ supervisor.transport().shutdown().join();
+
+ return returnValue;
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java
new file mode 100644
index 00000000000..f264fd13ddc
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.rpc;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.service.VespaService;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Test;
+
+import java.util.List;
+
+import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.rpc.IntegrationTester.CUSTOM_CONSUMER_ID;
+import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM;
+import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID;
+import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_2_CONFIG_ID;
+import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author jobergum
+ * @author gjoranv
+ */
+public class RpcMetricsTest {
+
+ private static final String METRICS_RESPONSE_CCL =
+ TestUtil.getContents("metrics-storage-simple.json").trim();
+
+ // see factory/doc/port-ranges.txt
+ private static final int httpPort = 18633;
+ private static final int rpcPort = 18634;
+
+ @Test
+ public void testGetMetrics() throws Exception {
+ try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) {
+ tester.httpServer().setResponse(METRICS_RESPONSE_CCL);
+ List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID);
+
+ assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1));
+
+ VespaService qrserver = services.get(0);
+ assertThat(qrserver.getMonitoringName(), is(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver"));
+
+ Metrics metrics = qrserver.getMetrics();
+ assertThat("Fetched number of metrics is not correct", metrics.size(), is(2));
+ Metric m = metrics.getMetric("foo.count");
+ assertNotNull("Did not find expected metric with name 'foo.count'", m);
+ Metric m2 = metrics.getMetric("bar.count");
+ assertNotNull("Did not find expected metric with name 'bar.count'", m2);
+
+ // Setup RPC client
+ Supervisor supervisor = new Supervisor(new Transport());
+ Target target = supervisor.connect(new Spec("localhost", rpcPort));
+
+ verifyMetricsFromRpcRequest(qrserver, target);
+
+ services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID);
+ assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1));
+
+ VespaService storageService = services.get(0);
+ verfiyMetricsFromServiceObject(storageService);
+
+ String metricsById = getMetricsById(storageService.getConfigId(), target);
+ assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 "));
+
+ String jsonResponse = getMetricsForYamas("non-existing", target).trim();
+ assertThat(jsonResponse, is("105: No service with name 'non-existing'"));
+
+ verifyMetricsFromRpcRequestForAllServices(target);
+
+ // Shutdown RPC
+ target.close();
+ supervisor.transport().shutdown().join();
+ }
+ }
+
+ private static void verifyMetricsFromRpcRequest(VespaService service, Target target) throws JSONException {
+ String jsonResponse = getMetricsForYamas(service.getMonitoringName(), target).trim();
+ JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics");
+ assertThat("Expected 3 metric messages", metrics.length(), is(3));
+ for (int i = 0; i < metrics.length() - 1; i++) { // The last "metric message" contains only status code/message
+ JSONObject jsonObject = metrics.getJSONObject(i);
+ assertFalse(jsonObject.has("status_code"));
+ assertFalse(jsonObject.has("status_msg"));
+ assertThat(jsonObject.getJSONObject("dimensions").getString("foo"), is("bar"));
+ assertThat(jsonObject.getJSONObject("dimensions").getString("bar"), is("foo"));
+ assertThat(jsonObject.getJSONObject("dimensions").getString("serviceDim"), is("serviceDimValue"));
+ assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").length(), is(1));
+ if (jsonObject.getJSONObject("metrics").has("foo_count")) {
+ assertThat(jsonObject.getJSONObject("metrics").getInt("foo_count"), is(1));
+ assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(VESPA_CONSUMER_ID.id));
+ } else {
+ assertThat(jsonObject.getJSONObject("metrics").getInt("foo.count"), is(1));
+ assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(CUSTOM_CONSUMER_ID.id));
+ }
+ }
+
+ verifyStatusMessage(metrics.getJSONObject(metrics.length() - 1));
+ }
+
+ private void verfiyMetricsFromServiceObject(VespaService service) {
+ Metrics storageMetrics = service.getMetrics();
+ assertThat(storageMetrics.size(), is(2));
+ Metric foo = storageMetrics.getMetric("foo.count");
+ assertNotNull("Did not find expected metric with name 'foo.count'", foo);
+ assertThat("Expected 2 dimensions for metric foo", foo.getDimensions().size(), is(2));
+ assertThat("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo")), is(true));
+ assertThat("Metric foo did not contain correct dimension", foo.getDimensions().get(toDimensionId("foo")), is("bar"));
+ assertThat("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar")), is(true));
+ assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo"));
+ }
+
+ private void verifyMetricsFromRpcRequestForAllServices(Target target) throws JSONException {
+ // Verify that metrics for all services can be retrieved in one request.
+ String allServicesResponse = getMetricsForYamas(ALL_SERVICES, target).trim();
+ JSONArray allServicesMetrics = new JSONObject(allServicesResponse).getJSONArray("metrics");
+ assertThat(allServicesMetrics.length(), is(5));
+ }
+
+ @Test
+ public void testGetAllMetricNames() {
+ try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) {
+
+ tester.httpServer().setResponse(METRICS_RESPONSE_CCL);
+ List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID);
+
+ assertThat(services.size(), is(1));
+ Metrics metrics = services.get(0).getMetrics();
+ assertThat("Fetched number of metrics is not correct", metrics.size(), is(2));
+ Metric m = metrics.getMetric("foo.count");
+ assertNotNull("Did not find expected metric with name 'foo.count'", m);
+
+
+ Metric m2 = metrics.getMetric("bar.count");
+ assertNotNull("Did not find expected metric with name 'bar'", m2);
+
+ // Setup RPC
+ Supervisor supervisor = new Supervisor(new Transport());
+ Target target = supervisor.connect(new Spec("localhost", rpcPort));
+
+ String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, target);
+ assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,"));
+
+ // Shutdown RPC
+ target.close();
+ supervisor.transport().shutdown().join();
+ }
+ }
+
+ private static String getMetricsForYamas(String service, Target target) {
+ Request req = new Request("getMetricsForYamas");
+ req.parameters().add(new StringValue(service));
+ return invoke(req, target);
+ }
+
+ private String getMetricsById(String service, Target target) {
+ Request req = new Request("getMetricsById");
+ req.parameters().add(new StringValue(service));
+ return invoke(req, target);
+ }
+
+ private String getAllMetricNamesForService(String service, ConsumerId consumer, Target target) {
+ Request req = new Request("getAllMetricNamesForService");
+ req.parameters().add(new StringValue(service));
+ req.parameters().add(new StringValue(consumer.id));
+ return invoke(req, target);
+ }
+
+ private static String invoke(Request req, Target target) {
+ String returnValue;
+ target.invokeSync(req, 20.0);
+ if (req.checkReturnTypes("s")) {
+ returnValue = req.returnValues().get(0).asString();
+ } else {
+ System.out.println(req.methodName() + " from rpcserver - Invocation failed "
+ + req.errorCode() + ": " + req.errorMessage());
+ returnValue = req.errorCode() + ": " + req.errorMessage();
+ }
+ return returnValue;
+ }
+
+ private static void verifyStatusMessage(JSONObject jsonObject) throws JSONException {
+ assertThat(jsonObject.getInt("status_code"), is(0));
+ assertThat(jsonObject.getString("status_msg"), notNullValue());
+ assertThat(jsonObject.getString("application"), notNullValue());
+ assertThat(jsonObject.getString("routing"), notNullValue());
+ assertThat(jsonObject.length(), is(4));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java
new file mode 100644
index 00000000000..bd61b8443aa
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknown
+ */
+public class ConfigSentinelClientTest {
+
+ @Test
+ public void testConfigSentinelClient() {
+ ConfigSentinelDummy configsentinel = new ConfigSentinelDummy();
+ List<VespaService> services = new ArrayList<>();
+ VespaService docproc = new VespaService("docprocservice", "docproc/cluster.x.indexing/0");
+ VespaService searchnode4 = new VespaService("searchnode4", "search/cluster.x/g0/c1/r1");
+ VespaService qrserver = new VespaService("qrserver", "container/qrserver.0");
+
+ services.add(searchnode4);
+ services.add(qrserver);
+ services.add(docproc);
+
+ MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel);
+ client.updateServiceStatuses(services);
+
+ assertThat(qrserver.getPid(), is(6520));
+ assertThat(qrserver.getState(), is("RUNNING"));
+ assertThat(qrserver.isAlive(), is(true));
+ assertThat(searchnode4.getPid(), is(6534));
+ assertThat(searchnode4.getState(), is("RUNNING"));
+ assertThat(searchnode4.isAlive(), is(true));
+
+ assertThat(docproc.getPid(), is(-1));
+ assertThat(docproc.getState(), is("FINISHED"));
+ assertThat(docproc.isAlive(), is(false));
+
+
+ configsentinel.reConfigure();
+
+ client.ping(docproc);
+ assertThat(docproc.getPid(), is(100));
+ assertThat(docproc.getState(), is("RUNNING"));
+ assertThat(docproc.isAlive(), is(true));
+
+ //qrserver has yet not been checked
+ assertThat(qrserver.isAlive(), is(true));
+
+ client.updateServiceStatuses(services);
+
+ assertThat(docproc.getPid(), is(100));
+ assertThat(docproc.getState(), is("RUNNING"));
+ assertThat(docproc.isAlive(), is(true));
+ //qrserver is no longer running on this node - so should be false
+ assertThat(qrserver.isAlive(), is(false));
+ }
+
+ @Test
+ public void testElastic() throws Exception {
+ String response = "container state=RUNNING mode=AUTO pid=14338 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"get/container.0\"\n" +
+ "container-clustercontroller state=RUNNING mode=AUTO pid=25020 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/cluster-controllers/0\"\n" +
+ "distributor state=RUNNING mode=AUTO pid=25024 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/distributor/0\"\n" +
+ "docprocservice state=RUNNING mode=AUTO pid=11973 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.search.indexing/0\"\n" +
+ "logd state=RUNNING mode=AUTO pid=25016 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/logd\"\n" +
+ "logserver state=RUNNING mode=AUTO pid=25018 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" +
+ "metricsproxy state=RUNNING mode=AUTO pid=13107 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/metricsproxy\"\n" +
+ "searchnode state=RUNNING mode=AUTO pid=25023 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/0\"\n" +
+ "slobrok state=RUNNING mode=AUTO pid=25019 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" +
+ "topleveldispatch state=RUNNING mode=AUTO pid=25026 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/tlds/tld.0\"\n" +
+ "\n";
+
+ ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(response);
+ List<VespaService> services = new ArrayList<>();
+
+ VespaService container = VespaService.create("container", "get/container.0", -1);
+
+ VespaService containerClusterController =
+ VespaService.create("container-clustercontroller", "get/container.0", -1);
+
+ VespaService notPresent = VespaService.create("dummy","fake", -1);
+
+ services.add(container);
+ services.add(containerClusterController);
+ services.add(notPresent);
+
+ MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel);
+ client.updateServiceStatuses(services);
+ assertThat(container.isAlive(),is(true));
+ assertThat(container.getPid(),is(14338));
+ assertThat(container.getState(),is("RUNNING"));
+
+ assertThat(containerClusterController.isAlive(),is(true));
+ assertThat(containerClusterController.getPid(),is(25020));
+ assertThat(containerClusterController.getState(),is("RUNNING"));
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java
new file mode 100644
index 00000000000..108f5c18e1d
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+package ai.vespa.metricsproxy.service;
+
+/**
+ * @author Eirik Nygaard
+ */
+public class ConfigSentinelDummy {
+ private String serviceList =
+ "docprocservice state=FINISHED mode=MANUAL pid=6555 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n"
+ + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n"
+ + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n"
+ + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n"
+ + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n"
+ + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n"
+ + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n"
+ + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n"
+ + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n"
+ + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n"
+ + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n"
+ + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n"
+ + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n"
+ + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n"
+ + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n"
+ + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n"
+ + "qrserver state=RUNNING mode=AUTO pid=6520 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.0\"\n"
+ + "\n";
+
+
+ public ConfigSentinelDummy() {
+ }
+
+ public ConfigSentinelDummy(String response) {
+ serviceList = response;
+ }
+
+ public void reConfigure() {
+ this.serviceList = "docprocservice state=RUNNING mode=AUTO pid=100 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n"
+ + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n"
+ + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n"
+ + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n"
+ + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n"
+ + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n"
+ + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n"
+ + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n"
+ + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n"
+ + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n"
+ + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n"
+ + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n"
+ + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n"
+ + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n"
+ + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n"
+ + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n"
+ + "\n";
+ }
+
+ public String getServiceList() {
+ return serviceList;
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java
new file mode 100644
index 00000000000..4174b18f3a7
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.metric.Metric;
+import org.json.JSONException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknown
+ */
+public class ContainerServiceTest {
+
+ private MockHttpServer service;
+ private int csPort;
+
+ @BeforeClass
+ public static void init() {
+ HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests
+ }
+
+ @Before
+ public void setupHTTPServer() {
+ csPort = 18637; // see factory/doc/port-ranges.txt
+ try {
+ String response = TestUtil.getContents("metrics-container-state-multi-chain.json");
+ service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testMultipleQueryDimensions() throws JSONException {
+ int count = 0;
+ VespaService service = VespaService.create("service1", "id", csPort);
+ for (Metric m : service.getMetrics().getMetrics()) {
+ if (m.getName().equals("queries.rate")) {
+ count++;
+ System.out.println("Name: " + m.getName() + " value: " + m.getValue());
+ if (m.getDimensions().get(toDimensionId("chain")).equals("asvBlendingResult")) {
+ assertThat((Double)m.getValue(), is(26.4));
+ } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) {
+ assertThat((Double)m.getValue(), is(0.36666666666666664));
+ } else {
+ assertThat("Unknown unknown chain", false, is(true));
+ }
+ }
+ }
+ assertThat(count, is(2));
+ }
+
+ @After
+ public void shutdown() {
+ this.service.close();
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java
new file mode 100644
index 00000000000..380a992aead
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.Metrics;
+
+/**
+ * @author Unknown
+ */
+public class DummyService extends VespaService {
+ static final String NAME = "dummy";
+ public static final String METRIC_1 = "c.test";
+ public static final String METRIC_2 = "val";
+
+ private final int num;
+
+ public DummyService(int num, String configid) {
+ super(NAME, NAME + num, configid);
+ this.num = num;
+ }
+
+ @Override
+ public Metrics getMetrics() {
+ Metrics m = new Metrics();
+
+ long timestamp = System.currentTimeMillis() / 1000;
+ m.add(new Metric(METRIC_1, 5 * num + 1, timestamp));
+ m.add(new Metric(METRIC_2, 1.3 * num + 1.05, timestamp));
+
+ return m;
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java
new file mode 100644
index 00000000000..27e1bb97943
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.metric.Metrics;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknowm
+ */
+public class MetricsFetcherTest {
+ private static int port = 9; //port number is not used in this test
+
+ @Test
+ public void testStateFormatMetricsParse() {
+ String jsonData = TestUtil.getContents("metrics-state.json");
+ RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port);
+ Metrics metrics = fetcher.createMetrics(jsonData, 0);
+ assertThat("Wrong number of metrics", metrics.size(), is(10));
+ assertThat("Wrong value for metric", metrics.get("query_hits.count").intValue(), is(28));
+ assertThat("Wrong value for metric ", metrics.get("queries.rate").doubleValue(), is(0.4667));
+ assertThat("Wrong timestamp", metrics.getTimeStamp(), is(1334134700L));
+ }
+
+ @Test
+ public void testEmptyJson() {
+ String jsonData = "{}";
+ RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port);
+ Metrics metrics = fetcher.createMetrics(jsonData, 0);
+ assertThat("Wrong number of metrics", metrics.size(), is(0));
+ }
+
+ @Test
+ public void testErrors() {
+ String jsonData;
+ Metrics metrics;
+
+ RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port);
+
+ jsonData = "";
+ metrics = fetcher.createMetrics(jsonData, 0);
+ assertThat("Wrong number of metrics", metrics.size(), is(0));
+
+ jsonData = "{\n" +
+ "\"status\" : {\n" +
+ " \"code\" : \"up\",\n" +
+ " \"message\" : \"Everything ok here\"\n" +
+ "}\n" +
+ "}";
+ metrics = fetcher.createMetrics(jsonData, 0);
+ assertThat("Wrong number of metrics", metrics.size(), is(0));
+
+ jsonData = "{\n" +
+ "\"status\" : {\n" +
+ " \"code\" : \"up\",\n" +
+ " \"message\" : \"Everything ok here\"\n" +
+ "},\n" +
+ "\"metrics\" : {\n" +
+ " \"snapshot\" : {\n" +
+ " \"from\" : 1334134640.089,\n" +
+ " \"to\" : 1334134700.088\n" +
+ " },\n" +
+ " \"values\" : [\n" +
+ " {\n" +
+ " \"name\" : \"queries\",\n" +
+ " \"description\" : \"Number of queries executed during snapshot interval\",\n" +
+ " \"values\" : {\n" +
+ " \"count\" : null,\n" +
+ " \"rate\" : 0.4667\n" +
+ " },\n" +
+ " \"dimensions\" : {\n" +
+ " \"searcherid\" : \"x\"\n" +
+ " }\n" +
+ " }\n" + "" +
+ " ]\n" +
+ "}\n" +
+ "}";
+
+ metrics = fetcher.createMetrics(jsonData, 0);
+ assertThat("Wrong number of metrics", metrics.size(), is(0));
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java
new file mode 100644
index 00000000000..917c529e63e
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+package ai.vespa.metricsproxy.service;
+
+import com.yahoo.log.LogLevel;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Mock config sentinel
+ *
+ * @author hmusum
+ */
+public class MockConfigSentinelClient extends ConfigSentinelClient {
+ private final ConfigSentinelDummy configSentinel;
+ private final static Logger log = Logger.getLogger(MockConfigSentinelClient.class.getPackage().getName());
+
+ public MockConfigSentinelClient(ConfigSentinelDummy configSentinel) {
+ super();
+ this.configSentinel = configSentinel;
+ }
+
+ @Override
+ protected synchronized void setStatus(List<VespaService> services) throws Exception {
+ List<VespaService> updatedServices = new ArrayList<>();
+ String[] lines = configSentinel.getServiceList().split("\n");
+ for (String line : lines) {
+ if (line.equals("")) {
+ break;
+ }
+
+ VespaService s = parseServiceString(line, services);
+ if (s != null) {
+ updatedServices.add(s);
+ }
+ }
+
+ //Check if there are services that were not found in
+ //from the sentinel
+ for (VespaService s : services) {
+ if (!updatedServices.contains(s)) {
+ log.log(LogLevel.DEBUG, "Service " + s + " is no longer found with sentinel - setting alive = false");
+ s.setAlive(false);
+ }
+ }
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java
new file mode 100644
index 00000000000..fdf2fae3081
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+
+/**
+ * @author jobergum
+ */
+public class MockHttpServer {
+
+ private String response;
+ private HttpServer server;
+
+ /**
+ * Mock http server that will return response as body
+ *
+ * @param port the port to listen to
+ * @param response the response to return along with 200 OK
+ * @param path the file path that the server will accept requests for. E.g /state/v1/metrics
+ */
+ public MockHttpServer(int port, String response, String path) throws IOException {
+ this.response = response;
+ this.server = HttpServer.create(new InetSocketAddress(port), 10);
+ this.server.createContext(path, new MyHandler());
+ this.server.setExecutor(null); // creates a default executor
+ this.server.start();
+ System.out.println("Started web server on port " + port);
+ }
+
+ public synchronized void setResponse(String r) {
+ this.response = r;
+ }
+
+ public void close() {
+ this.server.stop(0);
+ }
+
+ private class MyHandler implements HttpHandler {
+ public void handle(HttpExchange t) throws IOException {
+ t.sendResponseHeaders(200, response.length());
+ OutputStream os = t.getResponseBody();
+ os.write(response.getBytes());
+ os.close();
+ }
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java
new file mode 100644
index 00000000000..a42d52b7ea6
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknown
+ */
+public class SystemPollerTest {
+
+ @Test
+ public void testSystemPoller() {
+ DummyService s = new DummyService(0, "id");
+ List<VespaService> services = new ArrayList<>();
+ services.add(s);
+
+ SystemPoller poller = new SystemPoller(services, 60*5);
+ assertThat(s.isAlive(), is(false));
+ services.remove(0);
+ poller.setServices(services);
+ long n = poller.getPidJiffies(s);
+ assertThat(n, is(0L));
+ long[] memusage = poller.getMemoryUsage(s);
+ assertThat(memusage[0], is(0L));
+ assertThat(memusage[1], is(0L));
+ }
+
+ @Test
+ public void testCPUJiffies() {
+ String line = "cpu1 102180864 789 56766899 12800020140 1654757 0 0";
+ CpuJiffies n = new CpuJiffies(line);
+ assertThat(n.getCpuId(), is(1));
+ assertThat(n.getTotalJiffies(), is(12960623449L));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java
new file mode 100644
index 00000000000..13be98db23a
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.metric.Metrics;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Unknown
+ */
+public class VespaServiceTest {
+ private MockHttpServer service;
+ private int csPort;
+ private static final String response;
+
+ static {
+ response = TestUtil.getContents("metrics-state.json");
+ HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests
+ }
+
+ @Before
+ public void setupHTTPServer() {
+ csPort = 18632; // see factory/doc/port-ranges.txt
+ try {
+ service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testService() {
+ VespaService service = new VespaService("qrserver", "container/qrserver.0");
+ assertThat(service.getServiceName(), is("qrserver"));
+ assertThat(service.getInstanceName(), is("qrserver"));
+ assertThat(service.getPid(), is(-1));
+ assertThat(service.getConfigId(), is("container/qrserver.0"));
+
+
+ service = VespaService.create("qrserver2", "container/qrserver.0", -1);
+ assertThat(service.getServiceName(), is("qrserver"));
+ assertThat(service.getInstanceName(), is("qrserver2"));
+ assertThat(service.getPid(), is(-1));
+ assertThat(service.getConfigId(), is("container/qrserver.0"));
+ }
+
+ @Test
+ // TODO: Make it possible to test this without running a HTTP server to create the response
+ public void testMetricsFetching() {
+ VespaService service = VespaService.create("service1", "id", csPort);
+ Metrics metrics = service.getMetrics();
+ assertThat(metrics.getMetric("queries.count").getValue().intValue(), is(28));
+
+ // Shutdown server and check that no metrics are returned (should use empty metrics
+ // when unable to fetch new metrics)
+ shutdown();
+
+ metrics = service.getMetrics();
+ assertThat(metrics.size(), is(0));
+ }
+
+ @After
+ public void shutdown() {
+ this.service.close();
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java
new file mode 100644
index 00000000000..bd0b670ca35
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.service;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import java.util.List;
+
+import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * TODO: add more tests
+ *
+ * @author gjoranv
+ */
+public class VespaServicesTest {
+
+ @Test
+ public void services_can_be_retrieved_from_monitoring_name() {
+ List<VespaService> dummyServices = ImmutableList.of(
+ new DummyService(0, "dummy/id/0"),
+ new DummyService(1, "dummy/id/1"));
+ VespaServices services = new VespaServices(dummyServices);
+
+ assertThat(services.getMonitoringServices("vespa.dummy").size(), is(2));
+ }
+
+ @Test
+ public void all_services_can_be_retrieved_by_using_special_name() {
+ List<VespaService> dummyServices = ImmutableList.of(
+ new DummyService(0, "dummy/id/0"));
+ VespaServices services = new VespaServices(dummyServices);
+
+ assertThat(services.getMonitoringServices(ALL_SERVICES).size(), is(1));
+ }
+
+}
diff --git a/metrics-proxy/src/test/resources/health-check-failed.response.json b/metrics-proxy/src/test/resources/health-check-failed.response.json
new file mode 100644
index 00000000000..e118f10ec5e
--- /dev/null
+++ b/metrics-proxy/src/test/resources/health-check-failed.response.json
@@ -0,0 +1,7 @@
+{
+ "status": {
+ "code": "down",
+ "message":"SOMETHING FAILED"
+ },
+ "metrics": []
+ }
diff --git a/metrics-proxy/src/test/resources/health-check.response.json b/metrics-proxy/src/test/resources/health-check.response.json
new file mode 100644
index 00000000000..8e3858ec5d8
--- /dev/null
+++ b/metrics-proxy/src/test/resources/health-check.response.json
@@ -0,0 +1,6 @@
+{
+ "status": {
+ "code": "OK",
+ "message":"WORKING"
+ }
+ } \ No newline at end of file
diff --git a/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json b/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json
new file mode 100644
index 00000000000..76d0be50cca
--- /dev/null
+++ b/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json
@@ -0,0 +1,281 @@
+{
+ "metrics": {
+ "snapshot": {
+ "from": 1.383132197389E9,
+ "to": 1.383132257389E9
+ },
+ "values": [
+ {
+ "name": "search_connections",
+ "values": {
+ "average": 3.459204315576534,
+ "count": 1,
+ "last": 3.459204315576534,
+ "max": 3.459204315576534,
+ "min": 3.459204315576534,
+ "rate": 0.016666666666666666
+ }
+ },
+ {
+ "name": "active_queries",
+ "values": {
+ "average": 0,
+ "count": 1,
+ "last": 0,
+ "max": 0,
+ "min": 0,
+ "rate": 0.016666666666666666
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumFailedResponseWrites",
+ "values": {
+ "count": 85,
+ "rate": 1.4166666666666667
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumFailedResponses",
+ "values": {
+ "count": 8,
+ "rate": 0.13333333333333333
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumConnections",
+ "values": {
+ "count": 1630,
+ "rate": 27.166666666666668
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumSuccessfulResponses",
+ "values": {
+ "count": 1621,
+ "rate": 27.016666666666666
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNetworkLatency",
+ "values": {
+ "average": 0.11715958713775308,
+ "count": 20152,
+ "last": 0,
+ "max": 55,
+ "min": 0,
+ "rate": 335.8666666666667
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumSuccessfulResponseWrites",
+ "values": {
+ "count": 20152,
+ "rate": 335.8666666666667
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverTotalSuccessfulResponseLatency",
+ "values": {
+ "average": 90.88401253918495,
+ "count": 1595,
+ "last": 80,
+ "max": 233,
+ "min": 0,
+ "rate": 26.583333333333332
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverNumRequests",
+ "values": {
+ "count": 1633,
+ "rate": 27.216666666666665
+ }
+ },
+ {
+ "dimensions": {
+ "serverName": "qrs-server",
+ "serverPort": "4080"
+ },
+ "name": "serverTotalFailedResponseLatency",
+ "values": {
+ "average": 0.75,
+ "count": 8,
+ "last": 1,
+ "max": 1,
+ "min": 0,
+ "rate": 0.13333333333333333
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "query_latency",
+ "values": {
+ "average": 83.35949367088608,
+ "count": 1580,
+ "last": 61,
+ "max": 224,
+ "min": 12,
+ "rate": 26.333333333333332
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "max_query_latency",
+ "values": {
+ "average": 83.35949367088608,
+ "count": 1580,
+ "last": 61,
+ "max": 224,
+ "min": 12,
+ "rate": 26.333333333333332
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "peak_qps",
+ "values": {
+ "average": 25.87656434951563,
+ "count": 6,
+ "last": 23.681592039800993,
+ "max": 29.7659845295212,
+ "min": 23.681592039800993,
+ "rate": 0.1
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "queries",
+ "values": {
+ "count": 1584,
+ "rate": 26.4
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "mean_query_latency",
+ "values": {
+ "average": 83.35949367088608,
+ "count": 1580,
+ "last": 61,
+ "max": 224,
+ "min": 12,
+ "rate": 26.333333333333332
+ }
+ },
+ {
+ "dimensions": {"chain": "asvBlendingResult"},
+ "name": "hits_per_query",
+ "values": {
+ "average": 173.70126582278482,
+ "count": 1580,
+ "last": 175,
+ "max": 175,
+ "min": 5,
+ "rate": 26.333333333333332
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "query_latency",
+ "values": {
+ "average": 39.40909090909091,
+ "count": 22,
+ "last": 26,
+ "max": 174,
+ "min": 13,
+ "rate": 0.36666666666666664
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "max_query_latency",
+ "values": {
+ "average": 39.40909090909091,
+ "count": 22,
+ "last": 26,
+ "max": 174,
+ "min": 13,
+ "rate": 0.36666666666666664
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "peak_qps",
+ "values": {
+ "average": 0.5890415170417276,
+ "count": 3,
+ "last": 0.40488561981240295,
+ "max": 0.864528399757932,
+ "min": 0.40488561981240295,
+ "rate": 0.05
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "queries",
+ "values": {
+ "count": 22,
+ "rate": 0.36666666666666664
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "mean_query_latency",
+ "values": {
+ "average": 39.40909090909091,
+ "count": 22,
+ "last": 26,
+ "max": 174,
+ "min": 13,
+ "rate": 0.36666666666666664
+ }
+ },
+ {
+ "dimensions": {"chain": "blendingResult"},
+ "name": "hits_per_query",
+ "values": {
+ "average": 47.5,
+ "count": 22,
+ "last": 28,
+ "max": 176,
+ "min": 5,
+ "rate": 0.36666666666666664
+ }
+ }
+ ]
+ },
+ "status": {"code": "up"},
+ "time": 1383132269767
+} \ No newline at end of file
diff --git a/metrics-proxy/src/test/resources/metrics-state.json b/metrics-proxy/src/test/resources/metrics-state.json
new file mode 100644
index 00000000000..b7773e5fb8b
--- /dev/null
+++ b/metrics-proxy/src/test/resources/metrics-state.json
@@ -0,0 +1,42 @@
+{
+"status" : {
+ "code" : "up",
+ "message" : "Everything ok here"
+},
+"metrics" : {
+ "snapshot" : {
+ "from" : 1334134640.089,
+ "to" : 1334134700.088
+ },
+ "values" : [
+ {
+ "name" : "queries",
+ "description" : "Number of queries executed during snapshot interval",
+ "values" : {
+ "count" : 28,
+ "rate" : 0.4667
+ },
+ "dimensions" : {
+ "searcherid" : "x"
+ }
+ },
+ {
+ "name" : "query_hits",
+ "description" : "Number of documents matched per query during snapshot interval",
+ "values" : {
+ "count" : 28,
+ "rate" : 0.4667,
+ "average" : 128.3,
+ "min" : 0,
+ "max" : 10000,
+ "sum" : 3584,
+ "median" : 124.0,
+ "std_deviation": 5.43
+ },
+ "dimensions" : {
+ "searcherid" : "x"
+ }
+ }
+ ]
+}
+}
diff --git a/metrics-proxy/src/test/resources/metrics-storage-simple.json b/metrics-proxy/src/test/resources/metrics-storage-simple.json
new file mode 100644
index 00000000000..00715b52046
--- /dev/null
+++ b/metrics-proxy/src/test/resources/metrics-storage-simple.json
@@ -0,0 +1,38 @@
+{
+ "status" : {
+ "code" : "up",
+ "message": "All good"
+ },
+
+ "metrics" : {
+ "snapshot" : {
+ "from": 1335523285,
+ "to": 1335525685
+ },
+ "values": [
+
+ {
+ "name" : "foo",
+ "values" : {
+ "count" : 1
+ },
+ "dimensions" : {
+ "foo": "bar",
+ "bar" : "foo"
+ }
+ },
+
+ {
+ "name" : "bar",
+ "values" : {
+ "count" : 2
+ },
+ "dimensions" : {
+ "d0": "d0val",
+ "d1" : "d1val"
+ }
+ }
+ ]
+
+ }
+}
diff --git a/metrics-proxy/src/test/resources/rpc-json-output-check.json b/metrics-proxy/src/test/resources/rpc-json-output-check.json
new file mode 100644
index 00000000000..701a06d82b2
--- /dev/null
+++ b/metrics-proxy/src/test/resources/rpc-json-output-check.json
@@ -0,0 +1 @@
+{"metrics":[{"status_code":1,"application":"test-system.qrserver","dimensions":{"metrictype":"health","instance":"qrserver"},"status_msg":"SOMETHING FAILED"}]} \ No newline at end of file
diff --git a/metrics-proxy/src/test/resources/yamas-array-no-routing.json b/metrics-proxy/src/test/resources/yamas-array-no-routing.json
new file mode 100644
index 00000000000..8f21e8253b9
--- /dev/null
+++ b/metrics-proxy/src/test/resources/yamas-array-no-routing.json
@@ -0,0 +1,19 @@
+{
+ "status_code" : 0,
+ "timestamp" : 1400047900,
+ "application" : "vespa.searchnode",
+ "metrics" : {
+ "cpu" : 55.5555555555555,
+ "memory_virt" : 22222222222,
+ "memory_rss" : 5555555555
+ },
+ "dimensions" : {
+ "applicationName" : "app",
+ "tenantName" : "tenant",
+ "metrictype" : "system",
+ "instance" : "searchnode",
+ "applicationInstance" : "default",
+ "clustername" : "cluster"
+ },
+ "status_msg" : "Data collected successfully"
+}
diff --git a/metrics-proxy/src/test/resources/yamas-array.json b/metrics-proxy/src/test/resources/yamas-array.json
new file mode 100644
index 00000000000..c9293623b25
--- /dev/null
+++ b/metrics-proxy/src/test/resources/yamas-array.json
@@ -0,0 +1,26 @@
+{
+ "status_code" : 0,
+ "timestamp" : 1400047900,
+ "application" : "vespa.searchnode",
+ "metrics" : {
+ "cpu" : 55.5555555555555,
+ "memory_virt" : 22222222222,
+ "memory_rss" : 5555555555
+ },
+ "dimensions" : {
+ "applicationName" : "app",
+ "tenantName" : "tenant",
+ "metrictype" : "system",
+ "instance" : "searchnode",
+ "applicationInstance" : "default",
+ "clustername" : "cluster"
+ },
+ "routing" : {
+ "yamas" : {
+ "namespaces" : [
+ "Vespa"
+ ]
+ }
+ },
+ "status_msg" : "Data collected successfully"
+}